netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     // Non-test jobs use the default timeout value.
 412 }
 413
 414 def static getJobFolder(def scenario) {
 415     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 416         return 'jitstress'
 417     }
 418     if (scenario == 'illink') {
 419         return 'illink'
 420     }
 421     return ''
 422 }
 423
 424 def static getStressModeDisplayName(def scenario) {
 425     def displayStr = ''
 426     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 427         def prefixLength = 'COMPlus_'.length()
 428         if (k.length() >= prefixLength) {
 429             def modeName = k.substring(prefixLength, k.length())
 430             displayStr += ' ' + modeName + '=' + v
 431         }
 432     }
 433
 434     if (isCoreFxScenario(scenario)) {
 435         displayStr = ('CoreFx ' + displayStr).trim()
 436     }
 437
 438     return displayStr
 439 }
 440
 441 def static getR2RDisplayName(def scenario) {
 442     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 443     def displayStr = scenario
 444     def prefixLength = 'r2r_'.length()
 445     if (displayStr.length() >= prefixLength) {
 446         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 447     } else if (scenario == 'r2r') {
 448         displayStr = "R2R"
 449     }
 450     return displayStr
 451 }
 452
 453 // Generates the string for creating a file that sets environment variables
 454 // that makes it possible to run stress modes.  Writes the script to the file
 455 // specified by the stepScriptLocation parameter.
 456 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 457     def stepScript = ''
 458     if (os == 'Windows_NT') {
 459         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 460         // BUG?: it seems this is ignored, as this script will be run in an environment where
 461         //       environment variables will be discarded before this is used.
 462         def timeout = 1800000
 463         stepScript += "set __TestTimeout=${timeout}\r\n"
 464
 465         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 466         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 467
 468         // Create at least an empty script.
 469         stepScript += "echo. > ${stepScriptLocation}\r\n"
 470
 471         stressModeVars.each{ k, v ->
 472             // Write out what we are writing to the script file
 473             stepScript += "echo Setting ${k}=${v}\r\n"
 474             // Write out the set itself to the script file`
 475             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 476         }
 477     }
 478     else {
 479         stepScript += "echo Setting variables for ${stressModeName}\n"
 480         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 481         stressModeVars.each{ k, v ->
 482             // Write out what we are writing to the script file
 483             stepScript += "echo Setting ${k}=${v}\n"
 484             // Write out the set itself to the script file`
 485             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 486         }
 487         stepScript += "chmod +x ${stepScriptLocation}\n"
 488     }
 489
 490     return stepScript
 491 }
 492
 493 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 494 // Returns string of commands to do this.
 495 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 496     assert (os == 'Windows_NT')
 497     def stepScript = ''
 498     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 499     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 500
 501     // Display the resulting script. This is useful when looking at the output log file.
 502     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 503     stepScript += "type ${stepScriptLocation}\r\n"
 504
 505     return stepScript
 506 }
 507
 508 def static isNeedDocker(def architecture, def os, def isBuild) {
 509     if (isBuild) {
 510         if (architecture == 'x86' && os == 'Ubuntu') {
 511             return true
 512         }
 513         else if (architecture == 'arm') {
 514             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 515                 return true
 516             }
 517         }
 518     }
 519     else {
 520         if (architecture == 'x86' && os == 'Ubuntu') {
 521             return true
 522         }
 523     }
 524     return false
 525 }
 526
 527 def static getDockerImageName(def architecture, def os, def isBuild) {
 528     // We must change some docker private images to official later
 529     if (isBuild) {
 530         if (architecture == 'x86' && os == 'Ubuntu') {
 531             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 532         }
 533         else if (architecture == 'arm') {
 534             if (os == 'Ubuntu') {
 535                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 536             }
 537             else if (os == 'Ubuntu16.04') {
 538                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 539             }
 540             else if (os == 'Tizen') {
 541                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 542             }
 543         }
 544     }
 545     else {
 546         if (architecture == 'x86' && os == 'Ubuntu') {
 547             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 548         }
 549     }
 550     println("Unknown architecture to use docker: ${architecture} ${os}");
 551     assert false
 552 }
 553
 554 // Calculates the name of the build job based on some typical parameters.
 555 //
 556 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 557     // If the architecture is x64, do not add that info into the build name.
 558     // Need to change around some systems and other builds to pick up the right builds
 559     // to do that.
 560
 561     def suffix = scenario != 'default' ? "_${scenario}" : '';
 562     if (isBuildOnly) {
 563         suffix += '_bld'
 564     }
 565     def baseName = ''
 566     switch (architecture) {
 567         case 'x64':
 568             if (scenario == 'default') {
 569                 // For now we leave x64 off of the name for compatibility with other jobs
 570                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 571             }
 572             else if (scenario == 'formatting') {
 573                 // we don't care about the configuration for the formatting job. It runs all configs
 574                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 575             }
 576             else {
 577                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 578             }
 579             break
 580         case 'arm64':
 581             if (os.toLowerCase() == "windows_nt") {
 582                 // These are cross builds
 583                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 584             }
 585             else {
 586                 // Defaults to a small page size set of machines.
 587                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 588             }
 589             break
 590         case 'arm':
 591             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             break
 593         case 'armlb':
 594             // These are cross builds
 595             if (os == 'Tizen') {
 596                 // ABI: softfp
 597                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 598             }
 599             else {
 600                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 601             }
 602             break
 603         case 'x86':
 604         case 'x86_arm_altjit':
 605         case 'x64_arm64_altjit':
 606             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 607             break
 608         default:
 609             println("Unknown architecture: ${architecture}");
 610             assert false
 611             break
 612     }
 613
 614     return baseName + suffix
 615 }
 616
 617 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 618
 619     // Check scenario.
 620     switch (scenario) {
 621         case 'default':
 622             switch (architecture) {
 623                 case 'x64':
 624                 case 'x86':
 625                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 626                         Utilities.addPeriodicTrigger(job, '@daily')
 627                     }
 628                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 629                         Utilities.addGithubPushTrigger(job)
 630                     }
 631                     break
 632                 case 'arm':
 633                 case 'armlb':
 634                 case 'x86_arm_altjit':
 635                 case 'x64_arm64_altjit':
 636                     Utilities.addGithubPushTrigger(job)
 637                     break
 638                 case 'arm64':
 639                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 640                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 641                     break
 642                 default:
 643                     println("Unknown architecture: ${architecture}");
 644                     assert false
 645                     break
 646             }
 647             break
 648         case 'r2r':
 649             assert !(os in bidailyCrossList)
 650             // r2r gets a push trigger for checked/release
 651             if (configuration == 'Checked' || configuration == 'Release') {
 652                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 653                 if (architecture == 'x64' && os != 'OSX10.12') {
 654                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 655                     if (isFlowJob || os == 'Windows_NT') {
 656                         Utilities.addGithubPushTrigger(job)
 657                     }
 658                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 659                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 660                     if (isFlowJob) {
 661                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 662                     }
 663                 }
 664                 // For x86, only add per-commit jobs for Windows
 665                 else if (architecture == 'x86') {
 666                     if (os == 'Windows_NT') {
 667                         Utilities.addGithubPushTrigger(job)
 668                     }
 669                 }
 670                 // arm64 r2r jobs should only run daily.
 671                 else if (architecture == 'arm64') {
 672                     if (os == 'Windows_NT') {
 673                         Utilities.addPeriodicTrigger(job, '@daily')
 674                     }
 675                 }
 676             }
 677             break
 678         case 'r2r_jitstress1':
 679         case 'r2r_jitstress2':
 680         case 'r2r_jitstressregs1':
 681         case 'r2r_jitstressregs2':
 682         case 'r2r_jitstressregs3':
 683         case 'r2r_jitstressregs4':
 684         case 'r2r_jitstressregs8':
 685         case 'r2r_jitstressregs0x10':
 686         case 'r2r_jitstressregs0x80':
 687         case 'r2r_jitstressregs0x1000':
 688         case 'r2r_jitminopts':
 689         case 'r2r_jitforcerelocs':
 690         case 'r2r_gcstress15':
 691             assert !(os in bidailyCrossList)
 692
 693             // GCStress=C is currently not supported on OS X
 694             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 695                 break
 696             }
 697
 698             // GC Stress 15 r2r gets a push trigger for checked/release
 699             if (configuration == 'Checked' || configuration == 'Release') {
 700                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 701                 if (architecture == 'x64') {
 702                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 703                     if (isFlowJob || os == 'Windows_NT') {
 704                         // Add a weekly periodic trigger
 705                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 706                     }
 707                 }
 708                 // For x86, only add per-commit jobs for Windows
 709                 else if (architecture == 'x86') {
 710                     if (os == 'Windows_NT') {
 711                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 712                     }
 713                 }
 714             }
 715             break
 716         case 'longgc':
 717             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 718             assert configuration == 'Release'
 719             assert architecture == 'x64'
 720             Utilities.addPeriodicTrigger(job, '@daily')
 721             // TODO: Add once external email sending is available again
 722             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 723             break
 724         case 'gcsimulator':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'standalone_gc':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert (configuration == 'Release' || configuration == 'Checked')
 735             // TODO: Add once external email sending is available again
 736             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 737             Utilities.addPeriodicTrigger(job, '@daily')
 738             break
 739         case 'gc_reliability_framework':
 740             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 741             assert (configuration == 'Release' || configuration == 'Checked')
 742             // Only triggered by phrase.
 743             break
 744         case 'ilrt':
 745             assert !(os in bidailyCrossList)
 746             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 747             if (architecture == 'x64' && configuration == 'Release') {
 748                 // We don't expect to see a job generated except in these scenarios
 749                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 750                 if (isFlowJob || os == 'Windows_NT') {
 751                     Utilities.addPeriodicTrigger(job, '@daily')
 752                 }
 753             }
 754             break
 755         case 'jitdiff':
 756             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 757             assert configuration == 'Checked'
 758             assert (architecture == 'x64' || architecture == 'x86')
 759             Utilities.addGithubPushTrigger(job)
 760             break
 761         case 'formatting':
 762             assert (os == 'Windows_NT' || os == "Ubuntu")
 763             assert architecture == 'x64'
 764             Utilities.addGithubPushTrigger(job)
 765             break
 766         case 'jitstressregs1':
 767         case 'jitstressregs2':
 768         case 'jitstressregs3':
 769         case 'jitstressregs4':
 770         case 'jitstressregs8':
 771         case 'jitstressregs0x10':
 772         case 'jitstressregs0x80':
 773         case 'jitstressregs0x1000':
 774         case 'minopts':
 775         case 'forcerelocs':
 776         case 'jitstress1':
 777         case 'jitstress2':
 778         case 'jitstress2_jitstressregs1':
 779         case 'jitstress2_jitstressregs2':
 780         case 'jitstress2_jitstressregs3':
 781         case 'jitstress2_jitstressregs4':
 782         case 'jitstress2_jitstressregs8':
 783         case 'jitstress2_jitstressregs0x10':
 784         case 'jitstress2_jitstressregs0x80':
 785         case 'jitstress2_jitstressregs0x1000':
 786         case 'tailcallstress':
 787         case 'jitsse2only':
 788         case 'jitnosimd':
 789         case 'corefx_baseline':
 790         case 'corefx_minopts':
 791         case 'corefx_jitstress1':
 792         case 'corefx_jitstress2':
 793         case 'corefx_jitstressregs1':
 794         case 'corefx_jitstressregs2':
 795         case 'corefx_jitstressregs3':
 796         case 'corefx_jitstressregs4':
 797         case 'corefx_jitstressregs8':
 798         case 'corefx_jitstressregs0x10':
 799         case 'corefx_jitstressregs0x80':
 800         case 'corefx_jitstressregs0x1000':
 801         case 'zapdisable':
 802             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 803                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 804                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 805                     if (os == 'Windows_NT') {
 806                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 807                         Utilities.addPeriodicTrigger(job, '@weekly')
 808                     }
 809                 }
 810                 else {
 811                     Utilities.addPeriodicTrigger(job, '@daily')
 812                 }
 813             }
 814             break
 815         case 'heapverify1':
 816         case 'gcstress0x3':
 817             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 818                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 819                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 820                     if (os == 'Windows_NT') {
 821                         Utilities.addPeriodicTrigger(job, '@daily')
 822                     }
 823                     // TODO: Add once external email sending is available again
 824                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 825                 }
 826                 else {
 827                     Utilities.addPeriodicTrigger(job, '@weekly')
 828                 }
 829             }
 830             break
 831         case 'gcstress0xc':
 832         case 'gcstress0xc_zapdisable':
 833         case 'gcstress0xc_zapdisable_jitstress2':
 834         case 'gcstress0xc_zapdisable_heapverify1':
 835         case 'gcstress0xc_jitstress1':
 836         case 'gcstress0xc_jitstress2':
 837         case 'gcstress0xc_minopts_heapverify1':
 838             // GCStress=C is currently not supported on OS X
 839             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 840                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 841                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 842                     // TODO: Enable a periodic trigger after tests are updated.
 843                     // Utilities.addPeriodicTrigger(job, '@daily')
 844                     // TODO: Add once external email sending is available again
 845                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 846                 }
 847                 else {
 848                     Utilities.addPeriodicTrigger(job, '@weekly')
 849                 }
 850             }
 851             break
 852
 853         case 'illink':
 854             // Testing on other operating systems TBD
 855             assert (os == 'Windows_NT' || os == 'Ubuntu')
 856             if (architecture == 'x64' || architecture == 'x86') {
 857                 if (configuration == 'Checked') {
 858                     Utilities.addPeriodicTrigger(job, '@daily')
 859                 }
 860             }
 861             break
 862
 863         case 'tieredcompilation':
 864         case 'corefx_tieredcompilation':
 865             // No periodic jobs just yet, still testing
 866             break
 867
 868         default:
 869             println("Unknown scenario: ${scenario}");
 870             assert false
 871             break
 872     }
 873     return
 874 }
 875
 876 // **************************
 877 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 878 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 879 // builds will do a couple extra steps.
 880 // **************************
 881
 882 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 883 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 884 // it's just used for internal builds
 885 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 886 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 887     if (isWindowsBuildOnlyJob) {
 888         return
 889     }
 890
 891     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 892     // Non pull request builds.
 893     if (!isPR) {
 894         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 895         return
 896     }
 897
 898      def arm64Users = [
 899         'AndyAyersMS',
 900         'briansull',
 901         'BruceForstall',
 902         'CarolEidt',
 903         'cmckinsey',
 904         'erozenfeld',
 905         'janvorli',
 906         'jashook',
 907         'JosephTremoulet',
 908         'pgodeq',
 909         'pgavlin',
 910         'rartemev',
 911         'russellhadley',
 912         'RussKeldorph',
 913         'sandreenko',
 914         'sdmaclea',
 915         'swaroop-sridhar',
 916         'jkotas',
 917         'markwilkie',
 918         'weshaggard'
 919     ]
 920
 921     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 922     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 923     def osGroup = getOSGroup(os)
 924     switch (architecture) {
 925         case 'x64': // editor brace matching: {
 926             if (scenario == 'formatting') {
 927                 assert configuration == 'Checked'
 928                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 929                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 930                 }
 931                 break
 932             }
 933
 934             switch (os) {
 935                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 936                 case 'Debian8.4':
 937                 case 'RHEL7.2':
 938                     if (scenario == 'default') {
 939                         assert !isFlowJob
 940                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 941                     }
 942                     break
 943                 case 'Ubuntu16.04':
 944                     assert !isFlowJob
 945                     assert scenario == 'default'
 946                     // Distinguish with the other architectures (arm and x86)
 947                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 948                     break
 949                 case 'Fedora24':
 950                 case 'Ubuntu16.10':
 951                     assert !isFlowJob
 952                     assert scenario == 'default'
 953                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 954                     break
 955                 case 'Ubuntu':
 956                     if (scenario == 'illink') {
 957                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 958                         break
 959                     }
 960                     // fall through
 961                 case 'OSX10.12':
 962                     // Triggers on the non-flow jobs aren't necessary here
 963                     // Corefx testing uses non-flow jobs.
 964                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 965                         break
 966                     }
 967                     switch (scenario) {
 968                         case 'default':
 969                             // OSX uses checked for default PR tests
 970                             if (configuration == 'Checked') {
 971                                 // Default trigger
 972                                 assert !job.name.contains("centos")
 973                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 974                             }
 975                             break
 976                         case 'jitdiff':
 977                             if (configuration == 'Checked') {
 978                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 979                             }
 980                             break
 981                         case 'ilrt':
 982                             if (configuration == 'Release') {
 983                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 984                             }
 985                             break
 986                         case 'longgc':
 987                             if (configuration == 'Release') {
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'gcsimulator':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'standalone_gc':
 997                             if (configuration == 'Release' || configuration == 'Checked') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'gc_reliability_framework':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         default:
1007                             if (isJitStressScenario(scenario)) {
1008                                 def displayStr = getStressModeDisplayName(scenario)
1009                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1010                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1011                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1012                             }
1013                             else if (isR2RScenario(scenario)) {
1014                                 if (configuration == 'Release' || configuration == 'Checked') {
1015                                     def displayStr = getR2RDisplayName(scenario)
1016                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1017                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1018                                 }
1019                             }
1020                             else {
1021                                 println("Unknown scenario: ${scenario}");
1022                                 assert false
1023                             }
1024                             break
1025                     }
1026                     break
1027
1028                 case 'CentOS7.1':
1029                     switch (scenario) {
1030                         case 'default':
1031                             // CentOS uses checked for default PR tests while debug is build only
1032                             if (configuration == 'Debug') {
1033                                 // Default trigger
1034                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1035                             }
1036
1037                             // Make sure this is a flow job to get build and test.
1038                             if (configuration == 'Checked' && isFlowJob) {
1039                                 assert job.name.contains("flow")
1040                                 // Default trigger
1041                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1042                             }
1043                             break
1044                         default:
1045                             if (isR2RScenario(scenario)) {
1046                                 if (configuration == 'Release' || configuration == 'Checked') {
1047                                     def displayStr = getR2RDisplayName(scenario)
1048                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1049                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1050                                 }
1051                             }
1052                             break
1053                     }
1054                     break
1055
1056                 case 'Windows_NT':
1057                     switch (scenario) {
1058                         case 'default':
1059                             // Default trigger
1060                             if (configuration == 'Checked') {
1061                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1062                             }
1063                             break
1064                         case 'jitdiff':
1065                             if (configuration == 'Checked') {
1066                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1067                             }
1068                             break
1069                         case 'ilrt':
1070                             if (configuration == 'Release') {
1071                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1072                             }
1073                             break
1074                         case 'longgc':
1075                             if (configuration == 'Release') {
1076                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1077                             }
1078                             break
1079                         case 'gcsimulator':
1080                             if (configuration == 'Release') {
1081                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1082                             }
1083                             break
1084                         case 'standalone_gc':
1085                             if (configuration == 'Release' || configuration == 'Checked') {
1086                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1087                             }
1088                             break
1089                         case 'gc_reliability_framework':
1090                             if (configuration == 'Release' || configuration == 'Checked') {
1091                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1092                             }
1093                             break
1094                         case 'illink':
1095                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1096                             break
1097                         default:
1098                             if (isJitStressScenario(scenario)) {
1099                                 def displayStr = getStressModeDisplayName(scenario)
1100                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1101                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1102                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1103                             }
1104                             else if (isR2RScenario(scenario)) {
1105                                 if (configuration == 'Release' || configuration == 'Checked') {
1106                                     def displayStr = getR2RDisplayName(scenario)
1107                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1108                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1109                                 }
1110                             }
1111                             else {
1112                                 println("Unknown scenario: ${scenario}");
1113                                 assert false
1114                             }
1115                             break
1116                     }
1117                     break
1118                 default:
1119                     println("Unknown os: ${os}");
1120                     assert false
1121                     break
1122             }
1123             break
1124         // editor brace matching: }
1125         case 'armlb':
1126         case 'arm': // editor brace matching: {
1127             switch (os) {
1128                 case 'Ubuntu':
1129                 case 'Ubuntu16.04':
1130                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1131                         assert scenario == 'default'
1132                         job.with {
1133                             publishers {
1134                                 azureVMAgentPostBuildAction {
1135                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1136                                 }
1137                             }
1138                         }
1139                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1140                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1141                         }
1142                         else {
1143                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1144                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1145                         }
1146                     }
1147                     break
1148                 case 'Tizen':
1149                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1150                         architecture='armel'
1151                         job.with {
1152                             publishers {
1153                                 azureVMAgentPostBuildAction {
1154                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1155                                 }
1156                             }
1157                         }
1158                         // Removing the regex will cause this to run on each PR.
1159                         if (configuration == 'Checked') {
1160                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1161                         }
1162                         else {
1163                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1165                         }
1166                     }
1167                     break
1168                 case 'Windows_NT':
1169                     // Set up a private trigger
1170                     def contextString = "${os} ${architecture} Cross ${configuration}"
1171                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1172                     if (scenario == 'default') {
1173                         contextString += " Innerloop"
1174                         triggerString += "\\W+Innerloop"
1175                     }
1176                     else {
1177                         contextString += " ${scenario}"
1178                         triggerString += "\\W+${scenario}"
1179                     }
1180
1181                     if (configuration == 'Debug') {
1182                         contextString += " Build"
1183                         triggerString += "\\W+Build"
1184                     } else {
1185                         contextString += " Build and Test"
1186                         triggerString += "\\W+Build and Test"
1187                     }
1188
1189                     triggerString += ".*"
1190
1191                     switch (scenario) {
1192                         case 'default':
1193                             // Only Checked is a default trigger.
1194                             if (configuration == 'Checked')
1195                             {
1196                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1197                             }
1198                             else
1199                             {
1200                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1201                             }
1202                             break
1203                         default:
1204                             // Stress jobs will use this code path.
1205                             if (isArmWindowsScenario(scenario)) {
1206                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1207                             }
1208                             break
1209                     }
1210                     break
1211                 default:
1212                     println("NYI os: ${os}");
1213                     assert false
1214                     break
1215             }
1216             break
1217         // editor brace matching: }
1218         case 'arm64': // editor brace matching: {
1219             // Set up a private trigger
1220             def contextString = "${os} ${architecture} Cross ${configuration}"
1221             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1222             if (scenario == 'default') {
1223                 contextString += " Innerloop"
1224                 triggerString += "\\W+Innerloop"
1225             }
1226             else {
1227                 contextString += " ${scenario}"
1228                 triggerString += "\\W+${scenario}"
1229             }
1230
1231             if (configuration == 'Debug') {
1232                 contextString += " Build"
1233                 triggerString += "\\W+Build"
1234             } else {
1235                 contextString += " Build and Test"
1236                 triggerString += "\\W+Build and Test"
1237             }
1238
1239             triggerString += ".*"
1240
1241             switch (os) {
1242                 case 'Ubuntu':
1243                 case 'Ubuntu16.04':
1244                     switch (scenario) {
1245                         case 'default':
1246                             if (configuration == 'Debug' && !isFlowJob) {
1247                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1248                             }
1249                             else {
1250                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1251                             }
1252
1253                             break
1254                         default:
1255                             if (isR2RScenario(scenario)) {
1256                                 if (configuration == 'Checked' || configuration == 'Release') {
1257                                     def displayStr = getR2RDisplayName(scenario)
1258                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1259                                 }
1260                             }
1261                             break
1262                     }
1263                     break
1264                 case 'Windows_NT':
1265                     assert isArmWindowsScenario(scenario)
1266                     switch (scenario) {
1267                         case 'default':
1268                             if (configuration == 'Checked') {
1269                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1270                             }
1271                             else {
1272                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1273                             }
1274                             break
1275                         default:
1276                             // Stress jobs will use this code path.
1277                             if (isArmWindowsScenario(scenario)) {
1278                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1279                             }
1280                             break
1281                     }
1282                     break
1283                 default:
1284                     println("NYI os: ${os}");
1285                     assert false
1286                     break
1287             }
1288             break
1289         // editor brace matching: }
1290         case 'x86': // editor brace matching: {
1291             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1292             if (os == 'Ubuntu') {
1293                 // Triggers on the non-flow jobs aren't necessary here
1294                 if (!isFlowJob) {
1295                     break
1296                 }
1297                 // on-demand only for ubuntu x86
1298                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1299                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1300                 break
1301             }
1302             switch (scenario) {
1303                 case 'default':
1304                     if (configuration == 'Checked') {
1305                         assert !job.name.contains("centos")
1306                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1307                     }
1308                     else {
1309                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1310                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1311                     }
1312                     break
1313                 case 'ilrt':
1314                     if (configuration == 'Release') {
1315                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1316                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1317                     }
1318                     break
1319                 case 'longgc':
1320                     if (configuration == 'Release') {
1321                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1322                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1323                     }
1324                     break
1325                 case 'gcsimulator':
1326                     if (configuration == 'Release') {
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1328                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1329                     }
1330                     break
1331                 case 'standalone_gc':
1332                     if (configuration == 'Release' || configuration == 'Checked') {
1333                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1334                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1335                     }
1336                     break
1337                 case 'illink':
1338                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1339                     break
1340                 default:
1341                     if (isJitStressScenario(scenario)) {
1342                         def displayStr = getStressModeDisplayName(scenario)
1343                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1344                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1345                     }
1346                     else if (isR2RScenario(scenario)) {
1347                         if (configuration == 'Release' || configuration == 'Checked') {
1348                             def displayStr = getR2RDisplayName(scenario)
1349                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1350                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1351                         }
1352                     }
1353                     else {
1354                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1355                         assert false
1356                     }
1357                     break
1358             }
1359             break
1360          // editor brace matching: }
1361         case 'x64_arm64_altjit':
1362         case 'x86_arm_altjit': // editor brace matching: {
1363             assert (os == 'Windows_NT')
1364             switch (scenario) {
1365                 case 'default':
1366                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1367                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1368                     break
1369                 default:
1370                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1371                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1372                     break
1373             }
1374             break
1375         // editor brace matching: }
1376         default:
1377             println("Unknown architecture: ${architecture}");
1378             assert false
1379             break
1380     }
1381 }
1382
1383 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1384     def buildCommands = [];
1385     def osGroup = getOSGroup(os)
1386     def lowerConfiguration = configuration.toLowerCase()
1387
1388     def priority = '1'
1389     if (scenario == 'default' && isPR == true) {
1390         priority = '0'
1391     }
1392
1393     def enableCorefxTesting = isCoreFxScenario(scenario)
1394
1395     // Calculate the build steps, archival, and xunit results
1396     switch (os) {
1397         case 'Windows_NT': // editor brace matching: {
1398             switch (architecture) {
1399                 case 'x64':
1400                 case 'x86':
1401                 case 'x86_arm_altjit':
1402                 case 'x64_arm64_altjit':
1403                     def arch = architecture
1404                     def buildOpts = ''
1405                     if (architecture == 'x86_arm_altjit') {
1406                         arch = 'x86'
1407                     }
1408                     else if (architecture == 'x64_arm64_altjit') {
1409                         arch = 'x64'
1410                     }
1411
1412                     if (scenario == 'formatting') {
1413                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1414                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1415                         break
1416                     }
1417
1418                     if (scenario == 'illink') {
1419                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1420                     }
1421
1422                     // If it is a release build for windows, ensure PGO is used, else fail the build
1423                     if ((lowerConfiguration == 'release') &&
1424                         (scenario in Constants.basicScenarios) &&
1425                         (architecture != 'x86_arm_altjit') &&
1426                         (architecture != 'x64_arm64_altjit')) {
1427
1428                         buildOpts += ' -enforcepgo'
1429                     }
1430
1431                     if (enableCorefxTesting) {
1432                         buildOpts += ' skiptests';
1433                     } else {
1434                         buildOpts += " -priority=${priority}"
1435                     }
1436
1437                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1438                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1439                     // 35 characters long.
1440
1441                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1442
1443                     if (!isBuildOnly) {
1444                         def runtestArguments = ''
1445                         def testOpts = 'collectdumps'
1446
1447                         if (isR2RScenario(scenario)) {
1448
1449                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1450                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1451                             // to cause the tests to be crossgen'ed.
1452
1453                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1454                                 testOpts += ' crossgenaltjit protononjit.dll'
1455                             } else {
1456                                 testOpts += ' crossgen'
1457                             }
1458
1459                             testOpts += ' runcrossgentests'
1460
1461                             if (scenario == 'r2r_jitstress1') {
1462                                 testOpts += ' jitstress 1'
1463                             }
1464                             else if (scenario == 'r2r_jitstress2') {
1465                                 testOpts += ' jitstress 2'
1466                             }
1467                             else if (scenario == 'r2r_jitstressregs1') {
1468                                 testOpts += ' jitstressregs 1'
1469                             }
1470                             else if (scenario == 'r2r_jitstressregs2') {
1471                                 testOpts += ' jitstressregs 2'
1472                             }
1473                             else if (scenario == 'r2r_jitstressregs3') {
1474                                 testOpts += ' jitstressregs 3'
1475                             }
1476                             else if (scenario == 'r2r_jitstressregs4') {
1477                                 testOpts += ' jitstressregs 4'
1478                             }
1479                             else if (scenario == 'r2r_jitstressregs8') {
1480                                 testOpts += ' jitstressregs 8'
1481                             }
1482                             else if (scenario == 'r2r_jitstressregs0x10') {
1483                                 testOpts += ' jitstressregs 0x10'
1484                             }
1485                             else if (scenario == 'r2r_jitstressregs0x80') {
1486                                 testOpts += ' jitstressregs 0x80'
1487                             }
1488                             else if (scenario == 'r2r_jitstressregs0x1000') {
1489                                 testOpts += ' jitstressregs 0x1000'
1490                             }
1491                             else if (scenario == 'r2r_jitminopts') {
1492                                 testOpts += ' jitminopts'
1493                             }
1494                             else if (scenario == 'r2r_jitforcerelocs') {
1495                                 testOpts += ' jitforcerelocs'
1496                             }
1497                             else if (scenario == 'r2r_gcstress15') {
1498                                 testOpts += ' gcstresslevel 0xF'
1499                             }
1500                         }
1501                         else if (scenario == 'jitdiff') {
1502                             testOpts += ' jitdisasm crossgen'
1503                         }
1504                         else if (scenario == 'ilrt') {
1505                             testOpts += ' ilasmroundtrip'
1506                         }
1507                         else if (isLongGc(scenario)) {
1508                             testOpts += " ${scenario} sequential"
1509                         }
1510                         else if (scenario == 'standalone_gc') {
1511                             testOpts += ' gcname clrgc.dll'
1512                         }
1513                         else if (scenario == 'illink') {
1514                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1515                         }
1516
1517                         // If we are running a stress mode, we should write out the set of key
1518                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1519
1520                         def envScriptPath = ''
1521                         if (isJitStressScenario(scenario)) {
1522                             def buildCommandsStr = ''
1523                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1524                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1525                             if (architecture == 'x86_arm_altjit') {
1526                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1527                             }
1528                             else if (architecture == 'x64_arm64_altjit') {
1529                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1530                             }
1531
1532                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1533                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1534                             // into a single script.
1535                             buildCommands += buildCommandsStr
1536                         }
1537                         else if (architecture == 'x86_arm_altjit') {
1538                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1539                         }
1540                         else if (architecture == 'x64_arm64_altjit') {
1541                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1542                         }
1543                         if (envScriptPath != '') {
1544                             testOpts += " TestEnv ${envScriptPath}"
1545                         }
1546
1547                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1548
1549                         if (enableCorefxTesting) {
1550                             def workspaceRelativeFxRoot = "_/fx"
1551                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1552
1553                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1554
1555                             setTestJobTimeOut(newJob, scenario)
1556
1557                             // Archive and process (only) the test results
1558                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1559                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1560
1561                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1562                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1563                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1564                         }
1565                         else if (isGcReliabilityFramework(scenario)) {
1566                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1567                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1568                         }
1569                         else {
1570                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1571                         }
1572                     }
1573
1574                     if (!enableCorefxTesting) {
1575                         // Run the rest of the build
1576                         // Build the mscorlib for the other OS's
1577                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1578                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1579
1580                         if (arch == "x64") {
1581                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1582                         }
1583
1584                         // Zip up the tests directory so that we don't use so much space/time copying
1585                         // 10s of thousands of files around.
1586                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1587
1588                         if (!isJitStressScenario(scenario)) {
1589                             // For windows, pull full test results and test drops for x86/x64.
1590                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1591                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1592                         }
1593
1594                         if (scenario == 'jitdiff') {
1595                             // retrive jit-dasm output for base commit, and run jit-diff
1596                             if (!isBuildOnly) {
1597                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1598                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1599                             }
1600                         }
1601
1602                         if (!isBuildOnly) {
1603                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1604                             setTestJobTimeOut(newJob, scenario)
1605                         }
1606                     }
1607                     break
1608                 case 'armlb':
1609                 case 'arm':
1610                     assert isArmWindowsScenario(scenario)
1611
1612                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1613                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1614
1615                     // Set time out
1616                     setTestJobTimeOut(newJob, scenario)
1617
1618                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1619                     {
1620                         // Up the timeout for arm checked testing only.
1621                         // Keep the longer timeout for gcstress.
1622                         Utilities.setJobTimeout(newJob, 240)
1623                     }
1624
1625                     def buildArchitecture = 'arm'
1626
1627                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1628                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1629                     def armCrossgenOpt = ''
1630                     if (architecture == 'arm') {
1631                         armCrossgenOpt = '-altjitcrossgen'
1632                     }
1633
1634                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1635                     priority = '1'
1636
1637                     // This is now a build only job. Do not run tests. Use the flow job.
1638                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1639
1640                     // Zip up the tests directory so that we don't use so much space/time copying
1641                     // 10s of thousands of files around.
1642                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1643
1644                     // Add archival.
1645                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1646                     break
1647                 case 'arm64':
1648                     assert isArmWindowsScenario(scenario)
1649
1650                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1651                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1652
1653                     // Set time out
1654                     setTestJobTimeOut(newJob, scenario)
1655                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1656                     {
1657                         // Up the timeout for arm checked testing only.
1658                         // Keep the longer timeout for gcstress.
1659                         Utilities.setJobTimeout(newJob, 240)
1660                     }
1661
1662                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1663                     priority = '1'
1664
1665                     // This is now a build only job. Do not run tests. Use the flow job.
1666                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1667
1668                     // Zip up the tests directory so that we don't use so much space/time copying
1669                     // 10s of thousands of files around.
1670                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1671
1672                     // Add archival.
1673                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1674                     break
1675                 default:
1676                     println("Unknown architecture: ${architecture}");
1677                     assert false
1678                     break
1679             }
1680             break
1681         // editor brace matching: }
1682         case 'Ubuntu':
1683         case 'Ubuntu16.04':
1684         case 'Ubuntu16.10':
1685         case 'Debian8.4':
1686         case 'OSX10.12':
1687         case 'CentOS7.1':
1688         case 'RHEL7.2':
1689         case 'Tizen':
1690         case 'Fedora24': // editor brace matching: {
1691             switch (architecture) {
1692                 case 'x64':
1693                 case 'x86':
1694                     if (architecture == 'x86' && os == 'Ubuntu') {
1695                         // build and PAL test
1696                         def dockerImage = getDockerImageName(architecture, os, true)
1697                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1698                         dockerImage = getDockerImageName(architecture, os, false)
1699                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1700                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1701                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1702                         break
1703                     }
1704
1705                     if (scenario == 'formatting') {
1706                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1707                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1708                         break
1709                     }
1710
1711                     if (scenario == 'illink') {
1712                         assert(os == 'Ubuntu')
1713                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1714                     }
1715
1716                     if (!enableCorefxTesting) {
1717                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1718                         // only on supported OS platforms.
1719                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1720                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1721                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1722                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1723
1724                         // Set time out
1725                         setTestJobTimeOut(newJob, scenario)
1726                         // Basic archiving of the build
1727                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1728                         // And pal tests
1729                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1730                     }
1731                     else {
1732                         // Corefx stress testing
1733                         assert os == 'Ubuntu'
1734                         assert architecture == 'x64'
1735                         assert lowerConfiguration == 'checked'
1736                         assert isJitStressScenario(scenario)
1737
1738                         // Build coreclr
1739                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1740
1741                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1742                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1743
1744                         // Build and text corefx
1745                         def workspaceRelativeFxRoot = "_/fx"
1746                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1747
1748                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1749
1750                         setTestJobTimeOut(newJob, scenario)
1751
1752                         // Archive and process (only) the test results
1753                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1754                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1755                     }
1756                     break
1757                 case 'arm64':
1758                     if (!enableCorefxTesting) {
1759                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1760
1761                         // HACK -- Arm64 does not have corefx jobs yet.
1762                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1763                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1764                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1765                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1766
1767                         setTestJobTimeOut(newJob, scenario)
1768                         // Basic archiving of the build
1769                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1770                     }
1771                     break
1772                 case 'armlb':
1773                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1774                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1775
1776                     // default values for Ubuntu
1777                     def arm_abi="arm"
1778                     def linuxCodeName="trusty"
1779                     if (os == 'Ubuntu16.04') {
1780                         linuxCodeName="xenial"
1781                     }
1782                     else if (os == 'Tizen') {
1783                         arm_abi="armel"
1784                         linuxCodeName="tizen"
1785                     }
1786
1787                     // Unzip the Windows test binaries first. Exit with 0
1788                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1789
1790                     // Unpack the corefx binaries
1791                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1792                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1793                     if (os != 'Tizen') {
1794                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1795                     }
1796                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1797                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1798                     if (os == 'Ubuntu') {
1799                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1800                     }
1801
1802                     // Call the ARM CI script to cross build and test using docker
1803                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1804                     --mode=docker \\
1805                     --${arm_abi} \\
1806                     --linuxCodeName=${linuxCodeName} \\
1807                     --buildConfig=${lowerConfiguration} \\
1808                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1809                     --coreFxBinDir=./bin/CoreFxBinDir \\
1810                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1811
1812                     // Basic archiving of the build, no pal tests
1813                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1814                     break
1815                 default:
1816                     println("Unknown architecture: ${architecture}");
1817                     assert false
1818                     break
1819             }
1820             break
1821         // editor brace matching: }
1822         default:
1823             println("Unknown os: ${os}");
1824             assert false
1825             break
1826     } // os
1827
1828     return buildCommands
1829 }
1830
1831 // Additional scenario which can alter behavior
1832
1833 Constants.allScenarios.each { scenario ->
1834     [true, false].each { isPR ->
1835         Constants.architectureList.each { architecture ->
1836             Constants.configurationList.each { configuration ->
1837                 Constants.osList.each { os ->
1838                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1839                     // and reset the os to Windows_NT
1840                     def isBuildOnly = false
1841                     if (os == 'Windows_NT_BuildOnly') {
1842                         isBuildOnly = true
1843                         os = 'Windows_NT'
1844                     }
1845
1846                     // Tizen is only supported for arm legacy_backend architecture
1847                     if (os == 'Tizen' && architecture != 'armlb') {
1848                         return
1849                     }
1850
1851                     // Skip totally unimplemented (in CI) configurations.
1852                     switch (architecture) {
1853                         case 'arm64':
1854                             if (os == 'Ubuntu16.04') {
1855                                 os = 'Ubuntu'
1856                             }
1857
1858                             // Windows and Ubuntu only
1859                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1860                                 return
1861                             }
1862                             break
1863                         case 'armlb':
1864                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1865                                 return
1866                             }
1867                             break
1868                         case 'arm':
1869                             if (os != 'Windows_NT') {
1870                                 return
1871                             }
1872                             break
1873                         case 'x86':
1874                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1875                                 return
1876                             }
1877                             break
1878                         case 'x86_arm_altjit':
1879                         case 'x64_arm64_altjit':
1880                             if (os != 'Windows_NT') {
1881                                 return
1882                             }
1883                             break
1884                         case 'x64':
1885                             // Everything implemented
1886                             break
1887                         default:
1888                             println("Unknown architecture: ${architecture}")
1889                             assert false
1890                             break
1891                     }
1892
1893                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1894                     // with checked builds)
1895                     if (isJitStressScenario(scenario)) {
1896                         if (configuration != 'Checked') {
1897                             return
1898                         }
1899
1900                         // Since these are just execution time differences,
1901                         // skip platforms that don't execute the tests here (Windows_NT only)
1902                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1903                         if (!isEnabledOS || isBuildOnly) {
1904                             return
1905                         }
1906
1907                         switch (architecture) {
1908                             case 'x64':
1909                             case 'x86':
1910                             case 'x86_arm_altjit':
1911                             case 'x64_arm64_altjit':
1912                                 // x86 ubuntu: default only
1913                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1914                                     return
1915                                 }
1916                                 // Windows: Everything implemented
1917                                 break
1918
1919                             default:
1920                                 // arm, arm64, armlb: stress is handled through flow jobs.
1921                                 return
1922                         }
1923                     }
1924                     else if (isR2RScenario(scenario)) {
1925                         if (os != 'Windows_NT') {
1926                             return
1927                         }
1928                         // R2R test runs are not implemented for arm/armlb/arm64.
1929                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1930                             return
1931                         }
1932                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1933                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1934                             return
1935                         }
1936                     }
1937                     else {
1938                         // Skip scenarios
1939                         switch (scenario) {
1940                             case 'ilrt':
1941                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1942                                 // the default scenario build
1943                                 if (os != 'Windows_NT') {
1944                                     return
1945                                 }
1946                                 // Only x64 for now
1947                                 if (architecture != 'x64') {
1948                                     return
1949                                 }
1950                                 // Release only
1951                                 if (configuration != 'Release') {
1952                                     return
1953                                 }
1954                                 break
1955                             case 'jitdiff':
1956                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1957                                     return
1958                                 }
1959                                 if (architecture != 'x64') {
1960                                     return
1961                                 }
1962                                 if (configuration != 'Checked') {
1963                                     return
1964                                 }
1965                                 break
1966                             case 'longgc':
1967                             case 'gcsimulator':
1968                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1969                                     return
1970                                 }
1971                                 if (architecture != 'x64') {
1972                                     return
1973                                 }
1974                                 if (configuration != 'Release') {
1975                                     return
1976                                 }
1977                                 break
1978                             case 'gc_reliability_framework':
1979                             case 'standalone_gc':
1980                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1981                                     return
1982                                 }
1983
1984                                 if (architecture != 'x64') {
1985                                     return
1986                                 }
1987
1988                                 if (configuration != 'Release' && configuration != 'Checked') {
1989                                     return
1990                                 }
1991                                 break
1992                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1993                             case 'formatting':
1994                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1995                                     return
1996                                 }
1997                                 if (architecture != 'x64') {
1998                                     return
1999                                 }
2000                                 if (configuration != 'Checked') {
2001                                     return
2002                                 }
2003                                 if (isBuildOnly) {
2004                                     return
2005                                 }
2006                                 break
2007                             case 'illink':
2008                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2009                                     return
2010                                 }
2011                                 if (architecture != 'x64' && architecture != 'x86') {
2012                                     return
2013                                 }
2014                                 if (isBuildOnly) {
2015                                     return
2016                                 }
2017                                 break
2018                             case 'default':
2019                                 // Nothing skipped
2020                                 break
2021                             default:
2022                                 println("Unknown scenario: ${scenario}")
2023                                 assert false
2024                                 break
2025                         }
2026                     }
2027
2028                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2029                     // runtime behavior, not compile-time behavior, are not interesting.
2030                     switch (architecture) {
2031                         case 'x86_arm_altjit':
2032                         case 'x64_arm64_altjit':
2033                             if (isGCStressRelatedTesting(scenario)) {
2034                                 return
2035                             }
2036                             break
2037                         default:
2038                             break
2039                     }
2040
2041                     // Calculate names
2042                     def lowerConfiguration = configuration.toLowerCase()
2043                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2044                     def folderName = getJobFolder(scenario)
2045
2046                     // Create the new job
2047                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2048
2049                     def machineAffinityOptions = null
2050
2051                     if (os != 'Windows_NT') {
2052                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2053                     }
2054                     else {
2055                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2056                     }
2057
2058                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2059
2060                     // Add all the standard options
2061                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2062                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2063
2064                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2065                     def osGroup = getOSGroup(os)
2066
2067                     newJob.with {
2068                         steps {
2069                             if (os == 'Windows_NT') {
2070                                 buildCommands.each { buildCommand ->
2071                                     batchFile(buildCommand)
2072                                 }
2073                             }
2074                             else {
2075                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2076                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2077                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2078                                     // Define the Windows Tests and Corefx build job names
2079                                     def WindowTestsName = projectFolder + '/' +
2080                                                           Utilities.getFullJobName(project,
2081                                                                                    getJobName(lowerConfiguration,
2082                                                                                               'x64' ,
2083                                                                                               'windows_nt',
2084                                                                                               'default',
2085                                                                                               true),
2086                                                                                    false)
2087                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2088                                                        Utilities.getFolderName(branch)
2089
2090                                     // Copy the Windows test binaries and the Corefx build binaries
2091                                     copyArtifacts(WindowTestsName) {
2092                                         includePatterns('bin/tests/tests.zip')
2093                                         buildSelector {
2094                                             latestSuccessful(true)
2095                                         }
2096                                     }
2097
2098                                     def arm_abi = 'arm'
2099                                     def corefx_os = 'linux'
2100                                     if (os == 'Tizen') {
2101                                         arm_abi = 'armel'
2102                                         corefx_os = 'tizen'
2103                                     }
2104
2105                                     // Let's use release CoreFX to test checked CoreCLR,
2106                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2107                                     def corefx_lowerConfiguration = lowerConfiguration
2108                                     if ( lowerConfiguration == 'checked' ) {
2109                                         corefx_lowerConfiguration='release'
2110                                     }
2111
2112                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2113                                         includePatterns('bin/build.tar.gz')
2114                                         buildSelector {
2115                                             latestSuccessful(true)
2116                                         }
2117                                     }
2118                                 }
2119
2120                                 buildCommands.each { buildCommand ->
2121                                     shell(buildCommand)
2122                                 }
2123                             }
2124                         }
2125                     } // newJob.with
2126
2127                 } // os
2128             } // configuration
2129         } // architecture
2130     } // isPR
2131 } // scenario
2132
2133
2134 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2135 Constants.allScenarios.each { scenario ->
2136     [true, false].each { isPR ->
2137         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2138             Constants.crossList.each { os ->
2139                 if (architecture == 'arm64') {
2140                     if (os != "Ubuntu" && os != "Windows_NT") {
2141                         return
2142                     }
2143                 } else if (architecture == 'arm' || architecture == 'armlb') {
2144                     if (os != 'Windows_NT') {
2145                         return
2146                     }
2147                 }
2148                 else if (architecture == 'x86') {
2149                     if (os != "Ubuntu") {
2150                         return
2151                     }
2152                 }
2153
2154                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2155
2156                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2157                     return
2158                 }
2159
2160                 Constants.configurationList.each { configuration ->
2161
2162                     // First, filter based on OS.
2163
2164                     if (os == 'Windows_NT') {
2165                         if (!isArmWindowsScenario(scenario)) {
2166                             return
2167                         }
2168                     }
2169                     else {
2170                         // Non-Windows
2171                         if (architecture == 'arm64') {
2172                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2173                                 return
2174                             }
2175                         }
2176                         else if (architecture == 'x86') {
2177                             // Linux/x86 only want default test
2178                             if (scenario != 'default') {
2179                                 return
2180                             }
2181                         }
2182                     }
2183
2184                     // For CentOS, we only want Checked/Release builds.
2185                     if (os == 'CentOS7.1') {
2186                         if (configuration != 'Checked' && configuration != 'Release') {
2187                             return
2188                         }
2189                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2190                             return
2191                         }
2192                     }
2193
2194                     // For RedHat and Debian, we only do Release builds.
2195                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2196                         if (configuration != 'Release') {
2197                             return
2198                         }
2199                         if (scenario != 'default') {
2200                             return
2201                         }
2202                     }
2203
2204                     // Next, filter based on scenario.
2205
2206                     if (isJitStressScenario(scenario)) {
2207                         if (configuration != 'Checked') {
2208                             return
2209                         }
2210                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2211                         if (isCoreFxScenario(scenario)) {
2212                             return
2213                         }
2214                     }
2215                     else if (isR2RBaselineScenario(scenario)) {
2216                         if (configuration != 'Checked' && configuration != 'Release') {
2217                             return
2218                         }
2219                     }
2220                     else if (isR2RStressScenario(scenario)) {
2221                         if (configuration != 'Checked') {
2222                             return
2223                         }
2224                     }
2225                     else {
2226                         // Skip scenarios
2227                         switch (scenario) {
2228                             case 'ilrt':
2229                             case 'longgc':
2230                             case 'gcsimulator':
2231                                 // Long GC tests take a long time on non-Release builds
2232                                 // ilrt is also Release only
2233                                 if (configuration != 'Release') {
2234                                     return
2235                                 }
2236                                 break
2237                             case 'jitdiff':
2238                                 if (configuration != 'Checked') {
2239                                     return;
2240                                 }
2241                                 break
2242                             case 'gc_reliability_framework':
2243                             case 'standalone_gc':
2244                                 if (configuration != 'Release' && configuration != 'Checked') {
2245                                     return
2246                                 }
2247                                 break
2248                             case 'formatting':
2249                                 return
2250                             case 'illink':
2251                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2252                                     return
2253                                 }
2254                                 break
2255                             case 'default':
2256                                 // Nothing skipped
2257                                 break
2258                             default:
2259                                 println("Unknown scenario: ${scenario}")
2260                                 assert false
2261                                 break
2262                         }
2263                     }
2264
2265                     // Done filtering. Now, create the jobs.
2266
2267                     def lowerConfiguration = configuration.toLowerCase()
2268                     def osGroup = getOSGroup(os)
2269                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2270
2271                     def inputCoreCLRBuildName = projectFolder + '/' +
2272                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2273
2274                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2275                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2276                     def testBuildScenario = scenario
2277                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2278                         testBuildScenario = 'default'
2279                     }
2280
2281                     def inputWindowsTestBuildArch = architecture
2282                     if (architecture == "arm64" && os != "Windows_NT") {
2283                         // Use the x64 test build for arm64 unix
2284                         inputWindowsTestBuildArch = "x64"
2285                     }
2286
2287                     def inputWindowTestsBuildName = projectFolder + '/' +
2288                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2289
2290                     // Enable Server GC for Ubuntu PR builds
2291                     def serverGCString = ''
2292                     if (os == 'Ubuntu' && isPR) {
2293                         serverGCString = '--useServerGC'
2294                     }
2295
2296                     def testOpts = ''
2297
2298                     if (isR2RScenario(scenario)) {
2299
2300                         testOpts += ' --crossgen --runcrossgentests'
2301
2302                         if (scenario == 'r2r_jitstress1') {
2303                             testOpts += ' --jitstress=1'
2304                         }
2305                         else if (scenario == 'r2r_jitstress2') {
2306                             testOpts += ' --jitstress=2'
2307                         }
2308                         else if (scenario == 'r2r_jitstressregs1') {
2309                             testOpts += ' --jitstressregs=1'
2310                         }
2311                         else if (scenario == 'r2r_jitstressregs2') {
2312                             testOpts += ' --jitstressregs=2'
2313                         }
2314                         else if (scenario == 'r2r_jitstressregs3') {
2315                             testOpts += ' --jitstressregs=3'
2316                         }
2317                         else if (scenario == 'r2r_jitstressregs4') {
2318                             testOpts += ' --jitstressregs=4'
2319                         }
2320                         else if (scenario == 'r2r_jitstressregs8') {
2321                             testOpts += ' --jitstressregs=8'
2322                         }
2323                         else if (scenario == 'r2r_jitstressregs0x10') {
2324                             testOpts += ' --jitstressregs=0x10'
2325                         }
2326                         else if (scenario == 'r2r_jitstressregs0x80') {
2327                             testOpts += ' --jitstressregs=0x80'
2328                         }
2329                         else if (scenario == 'r2r_jitstressregs0x1000') {
2330                             testOpts += ' --jitstressregs=0x1000'
2331                         }
2332                         else if (scenario == 'r2r_jitminopts') {
2333                             testOpts += ' --jitminopts'
2334                         }
2335                         else if (scenario == 'r2r_jitforcerelocs') {
2336                             testOpts += ' --jitforcerelocs'
2337                         }
2338                         else if (scenario == 'r2r_gcstress15') {
2339                             testOpts += ' --gcstresslevel=0xF'
2340                         }
2341                     }
2342                     else if (scenario == 'jitdiff') {
2343                         testOpts += ' --jitdisasm --crossgen'
2344                     }
2345                     else if (scenario == 'illink') {
2346                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2347                     }
2348                     else if (isLongGc(scenario)) {
2349                         // Long GC tests behave very poorly when they are not
2350                         // the only test running (many of them allocate until OOM).
2351                         testOpts += ' --sequential'
2352
2353                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2354                         // for running long GC and GCSimulator tests, respectively. We don't use them
2355                         // here because using a playlist file produces much more readable output on the CI machines
2356                         // and reduces running time.
2357                         //
2358                         // The Long GC playlist contains all of the tests that are
2359                         // going to be run. The GCSimulator playlist contains all of
2360                         // the GC simulator tests.
2361                         if (scenario == 'longgc') {
2362                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2363                         }
2364                         else if (scenario == 'gcsimulator') {
2365                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2366                         }
2367                     }
2368                     else if (isGcReliabilityFramework(scenario)) {
2369                         testOpts += ' --build-overlay-only'
2370                     }
2371                     else if (scenario == 'standalone_gc') {
2372                         if (osGroup == 'OSX') {
2373                             testOpts += ' --gcname=libclrgc.dylib'
2374                         }
2375                         else if (osGroup == 'Linux') {
2376                             testOpts += ' --gcname=libclrgc.so'
2377                         }
2378                         else {
2379                             println("Unexpected OS group: ${osGroup} for os ${os}")
2380                             assert false
2381                         }
2382                     }
2383
2384                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2385
2386                     def folder = getJobFolder(scenario)
2387                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2388                         // Add parameters for the inputs
2389
2390                         if (windowsArmJob == true) {
2391                             parameters {
2392                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2393                             }
2394                         }
2395                         else {
2396                             parameters {
2397                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2398                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2399                             }
2400                         }
2401
2402                         steps {
2403                             // Set up the copies
2404
2405                             // Coreclr build containing the tests and mscorlib
2406
2407                             if (windowsArmJob != true) {
2408                                 copyArtifacts(inputWindowTestsBuildName) {
2409                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2410                                     buildSelector {
2411                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2412                                     }
2413                                 }
2414                             }
2415
2416                             // Coreclr build we are trying to test
2417
2418                             copyArtifacts(inputCoreCLRBuildName) {
2419                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2420                                 buildSelector {
2421                                     buildNumber('${CORECLR_BUILD}')
2422                                 }
2423                             }
2424
2425                             // Windows CoreCLR Arm(64) will restore corefx
2426                             // packages correctly.
2427                             //
2428                             // In addition, test steps are entirely different
2429                             // because we do not have a unified runner
2430                             if (windowsArmJob != true) {
2431                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2432
2433                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2434                                 def osJobName
2435                                 if (os == 'Ubuntu') {
2436                                     osJobName = 'ubuntu14.04'
2437                                 }
2438                                 else if (architecture == 'x86') {
2439                                     if (os == 'Ubuntu') {
2440                                         // Linux/x86 corefx jobs does not build managed yet
2441                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2442                                         osJobName = "linux_arm_cross"
2443                                     }
2444                                 }
2445                                 else {
2446                                     osJobName = os.toLowerCase()
2447                                 }
2448                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2449                                     includePatterns('bin/build.tar.gz')
2450                                     buildSelector {
2451                                         latestSuccessful(true)
2452                                     }
2453                                 }
2454
2455                                 shell("mkdir ./bin/CoreFxBinDir")
2456                                 // Unpack the corefx binaries
2457                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2458
2459                                 // HACK -- Arm64 does not have corefx jobs yet.
2460                                 // Clone corefx and build the native packages overwriting the x64 packages.
2461                                 if (architecture == 'arm64') {
2462                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2463                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2464                                 }
2465                                 else if (architecture == 'x86') {
2466                                     shell("mkdir ./bin/CoreFxNative")
2467
2468                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2469                                         includePatterns('bin/build.tar.gz')
2470                                         targetDirectory('bin/CoreFxNative')
2471                                         buildSelector {
2472                                             latestSuccessful(true)
2473                                         }
2474                                     }
2475
2476                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2477                                 }
2478
2479                                 // Unzip the tests first.  Exit with 0
2480                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2481
2482                                 // Execute the tests
2483                                 def runDocker = isNeedDocker(architecture, os, false)
2484                                 def dockerPrefix = ""
2485                                 def dockerCmd = ""
2486                                 if (runDocker) {
2487                                     def dockerImage = getDockerImageName(architecture, os, false)
2488                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2489                                     dockerCmd = dockerPrefix + "${dockerImage} "
2490                                 }
2491
2492                                 // If we are running a stress mode, we'll set those variables first
2493                                 def testEnvOpt = ""
2494                                 if (isJitStressScenario(scenario)) {
2495                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2496                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2497                                     shell("${createScriptCmds}")
2498                                     testEnvOpt = "--test-env=" + scriptFileName
2499                                 }
2500
2501                                 if (isGCStressRelatedTesting(scenario)) {
2502                                     shell('./init-tools.sh')
2503                                 }
2504
2505                                 shell("""${dockerCmd}./tests/runtest.sh \\
2506                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2507                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2508                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2509                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2510                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2511                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2512
2513                                 if (isGcReliabilityFramework(scenario)) {
2514                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2515                                     if (serverGCString != '') {
2516                                         if (runDocker) {
2517                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2518                                         }
2519                                         else {
2520                                             shell("export COMPlus_gcServer=1")
2521                                         }
2522                                     }
2523
2524                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2525                                 }
2526                             }
2527
2528                             else { // windowsArmJob == true
2529                                 // Unzip tests.
2530                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2531
2532                                 // Build the build commands
2533                                 def buildCommands = ""
2534
2535                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2536                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2537                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2538
2539                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2540                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2541
2542                                 // For all jobs
2543                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2544
2545                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2546                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2547
2548                                 // Arm(32) ryujit
2549                                 if (architecture == "arm") {
2550                                     // **This is an AltJit**
2551
2552                                     addEnvVariable("COMPlus_AltJit", "*")
2553                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2554                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2555                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2556                                 }
2557
2558                                 // If we are running a stress mode, we'll set those variables as well
2559                                 def stressValues = null
2560                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2561                                     if (isJitStressScenario(scenario)) {
2562                                         stressValues = Constants.jitStressModeScenarios[scenario]
2563                                     }
2564                                     else {
2565                                         stressValues = Constants.r2rStressScenarios[scenario]
2566                                     }
2567
2568                                     stressValues.each { key, value ->
2569                                         addEnvVariable(key, value)
2570                                     }
2571                                 }
2572
2573                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2574                                 // crossgen the framework assemblies, etc.
2575
2576                                 // Create the smarty command
2577                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2578                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2579                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2580
2581                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2582
2583                                 if (architecture == "arm") {
2584                                     addExclude("PROTOJIT_FAIL")
2585                                 }
2586
2587                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2588                                     def failTag = "JITSTRESS_FAIL"
2589                                     def excludeTag = "JITSTRESS_EXCLUDE"
2590
2591                                     if (scenario.contains('gc')) {
2592                                         failTag = "GCSTRESS_FAIL"
2593                                         excludeTag = "GCSTRESS_EXCLUDE"
2594                                     }
2595
2596                                     addArchSpecificExclude(architecture, failTag)
2597                                     addArchSpecificExclude(architecture, excludeTag)
2598                                 }
2599
2600                                 else {
2601                                     addExclude("pri1")
2602                                 }
2603
2604                                 smartyCommand += "/lstFile Tests.lst"
2605
2606                                 def testListArch = [
2607                                     'arm64': 'arm64',
2608                                     'arm': 'arm',
2609                                     'armlb': 'arm'
2610                                 ]
2611
2612                                 def archLocation = testListArch[architecture]
2613
2614                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2615                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2616                                 addCommand("${smartyCommand}")
2617
2618                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2619                                 // However, we also need to remove all the variables that were set during this batch file, so we
2620                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2621                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2622                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2623                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2624                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2625                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2626
2627                                 // ZIP up the smarty output, no matter what the smarty result.
2628                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2629
2630                                 addCommand("echo %errorlevel%")
2631                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2632
2633                                 // Use the smarty errorlevel as the script errorlevel.
2634                                 addCommand("exit /b %__save_smarty_errorlevel%")
2635
2636                                 batchFile(buildCommands)
2637                             }
2638                         }
2639                     }
2640
2641                     if (scenario == 'jitdiff') {
2642                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2643                     }
2644
2645                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2646                     if (os in ['Ubuntu']) {
2647                         SummaryBuilder summaries = new SummaryBuilder()
2648                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2649                         summaries.emit(newJob)
2650                     }
2651
2652                     def affinityOptions = null
2653
2654                     if (windowsArmJob == true) {
2655                         affinityOptions = [
2656                             "use_arm64_build_machine" : false
2657                         ]
2658                     }
2659
2660                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2661                         affinityOptions = [
2662                             "large_pages" : false
2663                         ]
2664                     }
2665
2666                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2667                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2668
2669                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2670                     //         Should the "architecture==arm64" include arm/armlb?
2671                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2672                     setTestJobTimeOut(newJob, scenario)
2673                     if (architecture == 'arm64') {
2674                         Utilities.setJobTimeout(newJob, 240)
2675                     }
2676
2677                     if (windowsArmJob != true) {
2678                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2679                     }
2680                     else {
2681                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2682
2683                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2684                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2685                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2686                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2687                     }
2688
2689                     // Create a build flow to join together the build and tests required to run this test.
2690                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2691                     // Linux CoreCLR test
2692                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2693                     def fullTestJobName = projectFolder + '/' + newJob.name
2694                     // Add a reference to the input jobs for report purposes
2695                     JobReport.Report.addReference(inputCoreCLRBuildName)
2696                     JobReport.Report.addReference(inputWindowTestsBuildName)
2697                     JobReport.Report.addReference(fullTestJobName)
2698                     def newFlowJob = null
2699
2700                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2701                         // Do not create the flow job for RHEL jobs.
2702                         return
2703                     }
2704
2705                     if (windowsArmJob == true) {
2706                         // For Windows arm jobs there is no reason to build a parallel test job.
2707                         // The product build supports building and archiving the tests.
2708
2709                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2710                         buildFlow("""
2711 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2712
2713 // And then build the test build
2714 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2715 """)
2716                         }
2717                     }
2718                     else {
2719                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2720                         buildFlow("""
2721 // Build the input jobs in parallel
2722 parallel (
2723 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2724 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2725 )
2726
2727 // And then build the test build
2728 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2729             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2730 """)
2731                         }
2732                     }
2733
2734                     // For the flow jobs set the machine affinity as x64 if an armarch.
2735                     def flowArch = architecture
2736
2737                     if (flowArch in validWindowsNTCrossArches) {
2738                         flowArch = 'x64'
2739                         affinityOptions = null
2740                     }
2741
2742                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2743                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2744                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2745                 } // configuration
2746             } // os
2747         } // architecture
2748     } // isPR
2749 } // scenario
2750
2751 JobReport.Report.generateJobReport(out)
2752
2753 // Make the call to generate the help job
2754 Utilities.createHelperJob(this, project, branch,
2755     "Welcome to the ${project} Repository",  // This is prepended to the help message
2756     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2757
2758 Utilities.addCROSSCheck(this, project, branch)