netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, isPR, architecture, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 412         Utilities.setJobTimeout(newJob, 240)
 413     }
 414     else if (!(scenario == 'default' && isPR == true)) {
 415         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 416         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 417         Utilities.setJobTimeout(newJob, 240)
 418     }
 419     // Non-test jobs use the default timeout value.
 420 }
 421
 422 def static getJobFolder(def scenario) {
 423     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 424         return 'jitstress'
 425     }
 426     if (scenario == 'illink') {
 427         return 'illink'
 428     }
 429     return ''
 430 }
 431
 432 def static getStressModeDisplayName(def scenario) {
 433     def displayStr = ''
 434     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 435         def prefixLength = 'COMPlus_'.length()
 436         if (k.length() >= prefixLength) {
 437             def modeName = k.substring(prefixLength, k.length())
 438             displayStr += ' ' + modeName + '=' + v
 439         }
 440     }
 441
 442     if (isCoreFxScenario(scenario)) {
 443         displayStr = ('CoreFx ' + displayStr).trim()
 444     }
 445
 446     return displayStr
 447 }
 448
 449 def static getR2RDisplayName(def scenario) {
 450     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 451     def displayStr = scenario
 452     def prefixLength = 'r2r_'.length()
 453     if (displayStr.length() >= prefixLength) {
 454         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 455     } else if (scenario == 'r2r') {
 456         displayStr = "R2R"
 457     }
 458     return displayStr
 459 }
 460
 461 // Generates the string for creating a file that sets environment variables
 462 // that makes it possible to run stress modes.  Writes the script to the file
 463 // specified by the stepScriptLocation parameter.
 464 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 465     def stepScript = ''
 466     if (os == 'Windows_NT') {
 467         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 468         // BUG?: it seems this is ignored, as this script will be run in an environment where
 469         //       environment variables will be discarded before this is used.
 470         def timeout = 1800000
 471         stepScript += "set __TestTimeout=${timeout}\r\n"
 472
 473         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 474         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 475
 476         // Create at least an empty script.
 477         stepScript += "echo. > ${stepScriptLocation}\r\n"
 478
 479         stressModeVars.each{ k, v ->
 480             // Write out what we are writing to the script file
 481             stepScript += "echo Setting ${k}=${v}\r\n"
 482             // Write out the set itself to the script file`
 483             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 484         }
 485     }
 486     else {
 487         stepScript += "echo Setting variables for ${stressModeName}\n"
 488         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 489         stressModeVars.each{ k, v ->
 490             // Write out what we are writing to the script file
 491             stepScript += "echo Setting ${k}=${v}\n"
 492             // Write out the set itself to the script file`
 493             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 494         }
 495         stepScript += "chmod +x ${stepScriptLocation}\n"
 496     }
 497
 498     return stepScript
 499 }
 500
 501 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 502 // Returns string of commands to do this.
 503 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 504     assert (os == 'Windows_NT')
 505     def stepScript = ''
 506     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 507     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 508
 509     // Display the resulting script. This is useful when looking at the output log file.
 510     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 511     stepScript += "type ${stepScriptLocation}\r\n"
 512
 513     return stepScript
 514 }
 515
 516 def static isNeedDocker(def architecture, def os, def isBuild) {
 517     if (isBuild) {
 518         if (architecture == 'x86' && os == 'Ubuntu') {
 519             return true
 520         }
 521         else if (architecture == 'arm') {
 522             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 523                 return true
 524             }
 525         }
 526     }
 527     else {
 528         if (architecture == 'x86' && os == 'Ubuntu') {
 529             return true
 530         }
 531     }
 532     return false
 533 }
 534
 535 def static getDockerImageName(def architecture, def os, def isBuild) {
 536     // We must change some docker private images to official later
 537     if (isBuild) {
 538         if (architecture == 'x86' && os == 'Ubuntu') {
 539             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 540         }
 541         else if (architecture == 'arm') {
 542             if (os == 'Ubuntu') {
 543                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 544             }
 545             else if (os == 'Ubuntu16.04') {
 546                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 547             }
 548             else if (os == 'Tizen') {
 549                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 550             }
 551         }
 552     }
 553     else {
 554         if (architecture == 'x86' && os == 'Ubuntu') {
 555             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 556         }
 557     }
 558     println("Unknown architecture to use docker: ${architecture} ${os}");
 559     assert false
 560 }
 561
 562 // Calculates the name of the build job based on some typical parameters.
 563 //
 564 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 565     // If the architecture is x64, do not add that info into the build name.
 566     // Need to change around some systems and other builds to pick up the right builds
 567     // to do that.
 568
 569     def suffix = scenario != 'default' ? "_${scenario}" : '';
 570     if (isBuildOnly) {
 571         suffix += '_bld'
 572     }
 573     def baseName = ''
 574     switch (architecture) {
 575         case 'x64':
 576             if (scenario == 'default') {
 577                 // For now we leave x64 off of the name for compatibility with other jobs
 578                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 579             }
 580             else if (scenario == 'formatting') {
 581                 // we don't care about the configuration for the formatting job. It runs all configs
 582                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 583             }
 584             else {
 585                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 586             }
 587             break
 588         case 'arm64':
 589             if (os.toLowerCase() == "windows_nt") {
 590                 // These are cross builds
 591                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             }
 593             else {
 594                 // Defaults to a small page size set of machines.
 595                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 596             }
 597             break
 598         case 'arm':
 599             // These are cross builds
 600             if (os == 'Tizen') {
 601                 // ABI: softfp
 602                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 603             }
 604             else {
 605                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 606             }
 607             break
 608         case 'armlb':
 609             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 610             break
 611         case 'x86':
 612         case 'x86_arm_altjit':
 613         case 'x64_arm64_altjit':
 614             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 615             break
 616         default:
 617             println("Unknown architecture: ${architecture}");
 618             assert false
 619             break
 620     }
 621
 622     return baseName + suffix
 623 }
 624
 625 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 626
 627     // Check scenario.
 628     switch (scenario) {
 629         case 'default':
 630             switch (architecture) {
 631                 case 'x64':
 632                 case 'x86':
 633                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 634                         Utilities.addPeriodicTrigger(job, '@daily')
 635                     }
 636                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 637                         Utilities.addGithubPushTrigger(job)
 638                     }
 639                     break
 640                 case 'arm':
 641                 case 'armlb':
 642                 case 'x86_arm_altjit':
 643                 case 'x64_arm64_altjit':
 644                     Utilities.addGithubPushTrigger(job)
 645                     break
 646                 case 'arm64':
 647                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 648                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 649                     break
 650                 default:
 651                     println("Unknown architecture: ${architecture}");
 652                     assert false
 653                     break
 654             }
 655             break
 656         case 'r2r':
 657             assert !(os in bidailyCrossList)
 658             // r2r gets a push trigger for checked/release
 659             if (configuration == 'Checked' || configuration == 'Release') {
 660                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 661                 if (architecture == 'x64' && os != 'OSX10.12') {
 662                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 663                     if (isFlowJob || os == 'Windows_NT') {
 664                         Utilities.addGithubPushTrigger(job)
 665                     }
 666                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 667                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 668                     if (isFlowJob) {
 669                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 670                     }
 671                 }
 672                 // For x86, only add per-commit jobs for Windows
 673                 else if (architecture == 'x86') {
 674                     if (os == 'Windows_NT') {
 675                         Utilities.addGithubPushTrigger(job)
 676                     }
 677                 }
 678                 // arm64 r2r jobs should only run daily.
 679                 else if (architecture == 'arm64') {
 680                     if (os == 'Windows_NT') {
 681                         Utilities.addPeriodicTrigger(job, '@daily')
 682                     }
 683                 }
 684             }
 685             break
 686         case 'r2r_jitstress1':
 687         case 'r2r_jitstress2':
 688         case 'r2r_jitstressregs1':
 689         case 'r2r_jitstressregs2':
 690         case 'r2r_jitstressregs3':
 691         case 'r2r_jitstressregs4':
 692         case 'r2r_jitstressregs8':
 693         case 'r2r_jitstressregs0x10':
 694         case 'r2r_jitstressregs0x80':
 695         case 'r2r_jitstressregs0x1000':
 696         case 'r2r_jitminopts':
 697         case 'r2r_jitforcerelocs':
 698         case 'r2r_gcstress15':
 699             assert !(os in bidailyCrossList)
 700
 701             // GCStress=C is currently not supported on OS X
 702             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 703                 break
 704             }
 705
 706             // GC Stress 15 r2r gets a push trigger for checked/release
 707             if (configuration == 'Checked' || configuration == 'Release') {
 708                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 709                 if (architecture == 'x64') {
 710                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 711                     if (isFlowJob || os == 'Windows_NT') {
 712                         // Add a weekly periodic trigger
 713                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 714                     }
 715                 }
 716                 // For x86, only add per-commit jobs for Windows
 717                 else if (architecture == 'x86') {
 718                     if (os == 'Windows_NT') {
 719                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 720                     }
 721                 }
 722             }
 723             break
 724         case 'longgc':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, '@daily')
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'gcsimulator':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert configuration == 'Release'
 735             assert architecture == 'x64'
 736             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 737             // TODO: Add once external email sending is available again
 738             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 739             break
 740         case 'standalone_gc':
 741             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 742             assert (configuration == 'Release' || configuration == 'Checked')
 743             // TODO: Add once external email sending is available again
 744             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 745             Utilities.addPeriodicTrigger(job, '@daily')
 746             break
 747         case 'gc_reliability_framework':
 748             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 749             assert (configuration == 'Release' || configuration == 'Checked')
 750             // Only triggered by phrase.
 751             break
 752         case 'ilrt':
 753             assert !(os in bidailyCrossList)
 754             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 755             if (architecture == 'x64' && configuration == 'Release') {
 756                 // We don't expect to see a job generated except in these scenarios
 757                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 758                 if (isFlowJob || os == 'Windows_NT') {
 759                     Utilities.addPeriodicTrigger(job, '@daily')
 760                 }
 761             }
 762             break
 763         case 'jitdiff':
 764             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 765             assert configuration == 'Checked'
 766             assert (architecture == 'x64' || architecture == 'x86')
 767             Utilities.addGithubPushTrigger(job)
 768             break
 769         case 'formatting':
 770             assert (os == 'Windows_NT' || os == "Ubuntu")
 771             assert architecture == 'x64'
 772             Utilities.addGithubPushTrigger(job)
 773             break
 774         case 'jitstressregs1':
 775         case 'jitstressregs2':
 776         case 'jitstressregs3':
 777         case 'jitstressregs4':
 778         case 'jitstressregs8':
 779         case 'jitstressregs0x10':
 780         case 'jitstressregs0x80':
 781         case 'jitstressregs0x1000':
 782         case 'minopts':
 783         case 'forcerelocs':
 784         case 'jitstress1':
 785         case 'jitstress2':
 786         case 'jitstress2_jitstressregs1':
 787         case 'jitstress2_jitstressregs2':
 788         case 'jitstress2_jitstressregs3':
 789         case 'jitstress2_jitstressregs4':
 790         case 'jitstress2_jitstressregs8':
 791         case 'jitstress2_jitstressregs0x10':
 792         case 'jitstress2_jitstressregs0x80':
 793         case 'jitstress2_jitstressregs0x1000':
 794         case 'tailcallstress':
 795         case 'jitsse2only':
 796         case 'jitnosimd':
 797         case 'corefx_baseline':
 798         case 'corefx_minopts':
 799         case 'corefx_jitstress1':
 800         case 'corefx_jitstress2':
 801         case 'corefx_jitstressregs1':
 802         case 'corefx_jitstressregs2':
 803         case 'corefx_jitstressregs3':
 804         case 'corefx_jitstressregs4':
 805         case 'corefx_jitstressregs8':
 806         case 'corefx_jitstressregs0x10':
 807         case 'corefx_jitstressregs0x80':
 808         case 'corefx_jitstressregs0x1000':
 809         case 'zapdisable':
 810             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 811                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 812                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 813                     if (os == 'Windows_NT') {
 814                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 815                         Utilities.addPeriodicTrigger(job, '@weekly')
 816                     }
 817                 }
 818                 else {
 819                     Utilities.addPeriodicTrigger(job, '@daily')
 820                 }
 821             }
 822             break
 823         case 'heapverify1':
 824         case 'gcstress0x3':
 825             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 826                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 827                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 828                     if (os == 'Windows_NT') {
 829                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 830                         Utilities.addPeriodicTrigger(job, '@weekly')
 831                     }
 832                     // TODO: Add once external email sending is available again
 833                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 834                 }
 835                 else {
 836                     Utilities.addPeriodicTrigger(job, '@weekly')
 837                 }
 838             }
 839             break
 840         case 'gcstress0xc':
 841         case 'gcstress0xc_zapdisable':
 842         case 'gcstress0xc_zapdisable_jitstress2':
 843         case 'gcstress0xc_zapdisable_heapverify1':
 844         case 'gcstress0xc_jitstress1':
 845         case 'gcstress0xc_jitstress2':
 846         case 'gcstress0xc_minopts_heapverify1':
 847             // GCStress=C is currently not supported on OS X
 848             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 849                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 850                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 851                     if (os == 'Windows_NT') {
 852                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 853                         Utilities.addPeriodicTrigger(job, '@weekly')
 854                     }
 855                     // TODO: Add once external email sending is available again
 856                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 857                 }
 858                 else {
 859                     Utilities.addPeriodicTrigger(job, '@weekly')
 860                 }
 861             }
 862             break
 863
 864         case 'illink':
 865             // Testing on other operating systems TBD
 866             assert (os == 'Windows_NT' || os == 'Ubuntu')
 867             if (architecture == 'x64' || architecture == 'x86') {
 868                 if (configuration == 'Checked') {
 869                     Utilities.addPeriodicTrigger(job, '@daily')
 870                 }
 871             }
 872             break
 873
 874         case 'tieredcompilation':
 875         case 'corefx_tieredcompilation':
 876             // No periodic jobs just yet, still testing
 877             break
 878
 879         default:
 880             println("Unknown scenario: ${scenario}");
 881             assert false
 882             break
 883     }
 884     return
 885 }
 886
 887 // **************************
 888 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 889 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 890 // builds will do a couple extra steps.
 891 // **************************
 892
 893 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 894 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 895 // it's just used for internal builds
 896 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 897 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 898     if (isWindowsBuildOnlyJob) {
 899         return
 900     }
 901
 902     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 903     // Non pull request builds.
 904     if (!isPR) {
 905         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 906         return
 907     }
 908
 909      def arm64Users = [
 910         'AndyAyersMS',
 911         'briansull',
 912         'BruceForstall',
 913         'CarolEidt',
 914         'cmckinsey',
 915         'erozenfeld',
 916         'janvorli',
 917         'jashook',
 918         'JosephTremoulet',
 919         'pgodeq',
 920         'pgavlin',
 921         'rartemev',
 922         'russellhadley',
 923         'RussKeldorph',
 924         'sandreenko',
 925         'sdmaclea',
 926         'swaroop-sridhar',
 927         'jkotas',
 928         'markwilkie',
 929         'weshaggard'
 930     ]
 931
 932     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 933     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 934     def osGroup = getOSGroup(os)
 935     switch (architecture) {
 936         case 'x64': // editor brace matching: {
 937             if (scenario == 'formatting') {
 938                 assert configuration == 'Checked'
 939                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 940                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 941                 }
 942                 break
 943             }
 944
 945             switch (os) {
 946                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 947                 case 'Debian8.4':
 948                 case 'RHEL7.2':
 949                     if (scenario == 'default') {
 950                         assert !isFlowJob
 951                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 952                     }
 953                     break
 954                 case 'Ubuntu16.04':
 955                     assert !isFlowJob
 956                     assert scenario == 'default'
 957                     // Distinguish with the other architectures (arm and x86)
 958                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 959                     break
 960                 case 'Fedora24':
 961                 case 'Ubuntu16.10':
 962                     assert !isFlowJob
 963                     assert scenario == 'default'
 964                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 965                     break
 966                 case 'Ubuntu':
 967                     if (scenario == 'illink') {
 968                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 969                         break
 970                     }
 971                     // fall through
 972                 case 'OSX10.12':
 973                     // Triggers on the non-flow jobs aren't necessary here
 974                     // Corefx testing uses non-flow jobs.
 975                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 976                         break
 977                     }
 978                     switch (scenario) {
 979                         case 'default':
 980                             // OSX uses checked for default PR tests
 981                             if (configuration == 'Checked') {
 982                                 // Default trigger
 983                                 assert !job.name.contains("centos")
 984                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 985                             }
 986                             break
 987                         case 'jitdiff':
 988                             if (configuration == 'Checked') {
 989                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 990                             }
 991                             break
 992                         case 'ilrt':
 993                             if (configuration == 'Release') {
 994                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 995                             }
 996                             break
 997                         case 'longgc':
 998                             if (configuration == 'Release') {
 999                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1000                             }
1001                             break
1002                         case 'gcsimulator':
1003                             if (configuration == 'Release') {
1004                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1005                             }
1006                             break
1007                         case 'standalone_gc':
1008                             if (configuration == 'Release' || configuration == 'Checked') {
1009                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1010                             }
1011                             break
1012                         case 'gc_reliability_framework':
1013                             if (configuration == 'Release' || configuration == 'Checked') {
1014                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1015                             }
1016                             break
1017                         default:
1018                             if (isJitStressScenario(scenario)) {
1019                                 def displayStr = getStressModeDisplayName(scenario)
1020                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1021                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1022                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1023                             }
1024                             else if (isR2RScenario(scenario)) {
1025                                 if (configuration == 'Release' || configuration == 'Checked') {
1026                                     def displayStr = getR2RDisplayName(scenario)
1027                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1028                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1029                                 }
1030                             }
1031                             else {
1032                                 println("Unknown scenario: ${scenario}");
1033                                 assert false
1034                             }
1035                             break
1036                     }
1037                     break
1038
1039                 case 'CentOS7.1':
1040                     switch (scenario) {
1041                         case 'default':
1042                             // CentOS uses checked for default PR tests while debug is build only
1043                             if (configuration == 'Debug') {
1044                                 // Default trigger
1045                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1046                             }
1047
1048                             // Make sure this is a flow job to get build and test.
1049                             if (configuration == 'Checked' && isFlowJob) {
1050                                 assert job.name.contains("flow")
1051                                 // Default trigger
1052                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1053                             }
1054                             break
1055                         default:
1056                             if (isR2RScenario(scenario)) {
1057                                 if (configuration == 'Release' || configuration == 'Checked') {
1058                                     def displayStr = getR2RDisplayName(scenario)
1059                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1060                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1061                                 }
1062                             }
1063                             break
1064                     }
1065                     break
1066
1067                 case 'Windows_NT':
1068                     switch (scenario) {
1069                         case 'default':
1070                             // Default trigger
1071                             if (configuration == 'Checked') {
1072                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1073                             }
1074                             break
1075                         case 'jitdiff':
1076                             if (configuration == 'Checked') {
1077                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1078                             }
1079                             break
1080                         case 'ilrt':
1081                             if (configuration == 'Release') {
1082                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1083                             }
1084                             break
1085                         case 'longgc':
1086                             if (configuration == 'Release') {
1087                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1088                             }
1089                             break
1090                         case 'gcsimulator':
1091                             if (configuration == 'Release') {
1092                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1093                             }
1094                             break
1095                         case 'standalone_gc':
1096                             if (configuration == 'Release' || configuration == 'Checked') {
1097                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1098                             }
1099                             break
1100                         case 'gc_reliability_framework':
1101                             if (configuration == 'Release' || configuration == 'Checked') {
1102                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1103                             }
1104                             break
1105                         case 'illink':
1106                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1107                             break
1108                         default:
1109                             if (isJitStressScenario(scenario)) {
1110                                 def displayStr = getStressModeDisplayName(scenario)
1111                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1112                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1113                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1114                             }
1115                             else if (isR2RScenario(scenario)) {
1116                                 if (configuration == 'Release' || configuration == 'Checked') {
1117                                     def displayStr = getR2RDisplayName(scenario)
1118                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1119                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1120                                 }
1121                             }
1122                             else {
1123                                 println("Unknown scenario: ${scenario}");
1124                                 assert false
1125                             }
1126                             break
1127                     }
1128                     break
1129                 default:
1130                     println("Unknown os: ${os}");
1131                     assert false
1132                     break
1133             }
1134             break
1135         // editor brace matching: }
1136         case 'armlb':
1137         case 'arm': // editor brace matching: {
1138             switch (os) {
1139                 case 'Ubuntu':
1140                 case 'Ubuntu16.04':
1141                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1142                         break
1143                     }
1144                     assert scenario == 'default'
1145                     job.with {
1146                         publishers {
1147                             azureVMAgentPostBuildAction {
1148                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1149                             }
1150                         }
1151                     }
1152                     if (configuration == 'Debug') {
1153                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1154                     }
1155                     else {
1156                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1157                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1158                     }
1159                     break
1160                 case 'Tizen':
1161                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1162                         break
1163                     }
1164                     architecture='armel'
1165                     job.with {
1166                         publishers {
1167                             azureVMAgentPostBuildAction {
1168                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1169                             }
1170                         }
1171                     }
1172                     if (configuration == 'Checked') {
1173                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1174                     }
1175                     else {
1176                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1177                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1178                     }
1179                     break
1180                 case 'Windows_NT':
1181                     // Triggers on the non-flow jobs aren't necessary here
1182                     if (!isFlowJob) {
1183                         break
1184                     }
1185
1186                     // Set up a private trigger
1187                     def contextString = "${os} ${architecture} Cross ${configuration}"
1188                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1189                     if (scenario == 'default') {
1190                         contextString += " Innerloop"
1191                         triggerString += "\\W+Innerloop"
1192                     }
1193                     else {
1194                         contextString += " ${scenario}"
1195                         triggerString += "\\W+${scenario}"
1196                     }
1197
1198                     if (configuration == 'Debug') {
1199                         contextString += " Build"
1200                         triggerString += "\\W+Build"
1201                     } else {
1202                         contextString += " Build and Test"
1203                         triggerString += "\\W+Build and Test"
1204                     }
1205
1206                     triggerString += ".*"
1207
1208                     switch (scenario) {
1209                         case 'default':
1210                             // Only Checked is a default trigger.
1211                             if (configuration == 'Checked')
1212                             {
1213                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1214                             }
1215                             else
1216                             {
1217                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1218                             }
1219                             break
1220                         default:
1221                             // Stress jobs will use this code path.
1222                             if (isArmWindowsScenario(scenario)) {
1223                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1224                             }
1225                             break
1226                     }
1227                     break
1228                 default:
1229                     println("NYI os: ${os}");
1230                     assert false
1231                     break
1232             }
1233             break
1234         // editor brace matching: }
1235         case 'arm64': // editor brace matching: {
1236             // Set up a private trigger
1237             def contextString = "${os} ${architecture} Cross ${configuration}"
1238             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1239             if (scenario == 'default') {
1240                 contextString += " Innerloop"
1241                 triggerString += "\\W+Innerloop"
1242             }
1243             else {
1244                 contextString += " ${scenario}"
1245                 triggerString += "\\W+${scenario}"
1246             }
1247
1248             if (configuration == 'Debug') {
1249                 contextString += " Build"
1250                 triggerString += "\\W+Build"
1251             } else {
1252                 contextString += " Build and Test"
1253                 triggerString += "\\W+Build and Test"
1254             }
1255
1256             triggerString += ".*"
1257
1258             switch (os) {
1259                 case 'Ubuntu':
1260                 case 'Ubuntu16.04':
1261                     switch (scenario) {
1262                         case 'default':
1263                             if (configuration == 'Debug' && !isFlowJob) {
1264                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1265                             }
1266                             else {
1267                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1268                             }
1269
1270                             break
1271                         default:
1272                             if (isR2RScenario(scenario)) {
1273                                 if (configuration == 'Checked' || configuration == 'Release') {
1274                                     def displayStr = getR2RDisplayName(scenario)
1275                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1276                                 }
1277                             }
1278                             break
1279                     }
1280                     break
1281                 case 'Windows_NT':
1282                     // Triggers on the non-flow jobs aren't necessary here
1283                     if (!isFlowJob) {
1284                         break
1285                     }
1286
1287                     assert isArmWindowsScenario(scenario)
1288                     switch (scenario) {
1289                         case 'default':
1290                             if (configuration == 'Checked') {
1291                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1292                             }
1293                             else {
1294                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1295                             }
1296                             break
1297                         default:
1298                             // Stress jobs will use this code path.
1299                             if (isArmWindowsScenario(scenario)) {
1300                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1301                             }
1302                             break
1303                     }
1304                     break
1305                 default:
1306                     println("NYI os: ${os}");
1307                     assert false
1308                     break
1309             }
1310             break
1311         // editor brace matching: }
1312         case 'x86': // editor brace matching: {
1313             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1314             if (os == 'Ubuntu') {
1315                 // Triggers on the non-flow jobs aren't necessary here
1316                 if (!isFlowJob) {
1317                     break
1318                 }
1319                 // on-demand only for ubuntu x86
1320                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1321                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1322                 break
1323             }
1324             switch (scenario) {
1325                 case 'default':
1326                     if (configuration == 'Checked') {
1327                         assert !job.name.contains("centos")
1328                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1329                     }
1330                     else {
1331                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1332                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1333                     }
1334                     break
1335                 case 'ilrt':
1336                     if (configuration == 'Release') {
1337                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1338                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1339                     }
1340                     break
1341                 case 'longgc':
1342                     if (configuration == 'Release') {
1343                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1344                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1345                     }
1346                     break
1347                 case 'gcsimulator':
1348                     if (configuration == 'Release') {
1349                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1350                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1351                     }
1352                     break
1353                 case 'standalone_gc':
1354                     if (configuration == 'Release' || configuration == 'Checked') {
1355                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1356                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     }
1358                     break
1359                 case 'illink':
1360                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1361                     break
1362                 default:
1363                     if (isJitStressScenario(scenario)) {
1364                         def displayStr = getStressModeDisplayName(scenario)
1365                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1366                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1367                     }
1368                     else if (isR2RScenario(scenario)) {
1369                         if (configuration == 'Release' || configuration == 'Checked') {
1370                             def displayStr = getR2RDisplayName(scenario)
1371                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1372                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1373                         }
1374                     }
1375                     else {
1376                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1377                         assert false
1378                     }
1379                     break
1380             }
1381             break
1382          // editor brace matching: }
1383         case 'x64_arm64_altjit':
1384         case 'x86_arm_altjit': // editor brace matching: {
1385             assert (os == 'Windows_NT')
1386             switch (scenario) {
1387                 case 'default':
1388                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1389                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1390                     break
1391                 default:
1392                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1393                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1394                     break
1395             }
1396             break
1397         // editor brace matching: }
1398         default:
1399             println("Unknown architecture: ${architecture}");
1400             assert false
1401             break
1402     }
1403 }
1404
1405 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1406     def buildCommands = [];
1407     def osGroup = getOSGroup(os)
1408     def lowerConfiguration = configuration.toLowerCase()
1409
1410     def priority = '1'
1411     if (scenario == 'default' && isPR == true) {
1412         priority = '0'
1413     }
1414
1415     def enableCorefxTesting = isCoreFxScenario(scenario)
1416
1417     // Calculate the build steps, archival, and xunit results
1418     switch (os) {
1419         case 'Windows_NT': // editor brace matching: {
1420             switch (architecture) {
1421                 case 'x64':
1422                 case 'x86':
1423                 case 'x86_arm_altjit':
1424                 case 'x64_arm64_altjit':
1425                     def arch = architecture
1426                     def buildOpts = ''
1427                     if (architecture == 'x86_arm_altjit') {
1428                         arch = 'x86'
1429                     }
1430                     else if (architecture == 'x64_arm64_altjit') {
1431                         arch = 'x64'
1432                     }
1433
1434                     if (scenario == 'formatting') {
1435                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1436                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1437                         break
1438                     }
1439
1440                     if (scenario == 'illink') {
1441                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1442                     }
1443
1444                     // If it is a release build for windows, ensure PGO is used, else fail the build
1445                     if ((lowerConfiguration == 'release') &&
1446                         (scenario in Constants.basicScenarios) &&
1447                         (architecture != 'x86_arm_altjit') &&
1448                         (architecture != 'x64_arm64_altjit')) {
1449
1450                         buildOpts += ' -enforcepgo'
1451                     }
1452
1453                     if (enableCorefxTesting) {
1454                         buildOpts += ' skiptests';
1455                     } else {
1456                         buildOpts += " -priority=${priority}"
1457                     }
1458
1459                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1460                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1461                     // 35 characters long.
1462
1463                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1464
1465                     if (!isBuildOnly) {
1466                         def runtestArguments = ''
1467                         def testOpts = 'collectdumps'
1468
1469                         if (isR2RScenario(scenario)) {
1470
1471                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1472                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1473                             // to cause the tests to be crossgen'ed.
1474
1475                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1476                                 testOpts += ' crossgenaltjit protononjit.dll'
1477                             } else {
1478                                 testOpts += ' crossgen'
1479                             }
1480
1481                             testOpts += ' runcrossgentests'
1482
1483                             if (scenario == 'r2r_jitstress1') {
1484                                 testOpts += ' jitstress 1'
1485                             }
1486                             else if (scenario == 'r2r_jitstress2') {
1487                                 testOpts += ' jitstress 2'
1488                             }
1489                             else if (scenario == 'r2r_jitstressregs1') {
1490                                 testOpts += ' jitstressregs 1'
1491                             }
1492                             else if (scenario == 'r2r_jitstressregs2') {
1493                                 testOpts += ' jitstressregs 2'
1494                             }
1495                             else if (scenario == 'r2r_jitstressregs3') {
1496                                 testOpts += ' jitstressregs 3'
1497                             }
1498                             else if (scenario == 'r2r_jitstressregs4') {
1499                                 testOpts += ' jitstressregs 4'
1500                             }
1501                             else if (scenario == 'r2r_jitstressregs8') {
1502                                 testOpts += ' jitstressregs 8'
1503                             }
1504                             else if (scenario == 'r2r_jitstressregs0x10') {
1505                                 testOpts += ' jitstressregs 0x10'
1506                             }
1507                             else if (scenario == 'r2r_jitstressregs0x80') {
1508                                 testOpts += ' jitstressregs 0x80'
1509                             }
1510                             else if (scenario == 'r2r_jitstressregs0x1000') {
1511                                 testOpts += ' jitstressregs 0x1000'
1512                             }
1513                             else if (scenario == 'r2r_jitminopts') {
1514                                 testOpts += ' jitminopts'
1515                             }
1516                             else if (scenario == 'r2r_jitforcerelocs') {
1517                                 testOpts += ' jitforcerelocs'
1518                             }
1519                             else if (scenario == 'r2r_gcstress15') {
1520                                 testOpts += ' gcstresslevel 0xF'
1521                             }
1522                         }
1523                         else if (scenario == 'jitdiff') {
1524                             testOpts += ' jitdisasm crossgen'
1525                         }
1526                         else if (scenario == 'ilrt') {
1527                             testOpts += ' ilasmroundtrip'
1528                         }
1529                         else if (isLongGc(scenario)) {
1530                             testOpts += " ${scenario} sequential"
1531                         }
1532                         else if (scenario == 'standalone_gc') {
1533                             testOpts += ' gcname clrgc.dll'
1534                         }
1535                         else if (scenario == 'illink') {
1536                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1537                         }
1538
1539                         // If we are running a stress mode, we should write out the set of key
1540                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1541
1542                         def envScriptPath = ''
1543                         if (isJitStressScenario(scenario)) {
1544                             def buildCommandsStr = ''
1545                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1546                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1547                             if (architecture == 'x86_arm_altjit') {
1548                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1549                             }
1550                             else if (architecture == 'x64_arm64_altjit') {
1551                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1552                             }
1553
1554                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1555                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1556                             // into a single script.
1557                             buildCommands += buildCommandsStr
1558                         }
1559                         else if (architecture == 'x86_arm_altjit') {
1560                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1561                         }
1562                         else if (architecture == 'x64_arm64_altjit') {
1563                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1564                         }
1565                         if (envScriptPath != '') {
1566                             testOpts += " TestEnv ${envScriptPath}"
1567                         }
1568
1569                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1570
1571                         if (enableCorefxTesting) {
1572                             def workspaceRelativeFxRoot = "_/fx"
1573                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1574
1575                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1576
1577                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1578
1579                             // Archive and process (only) the test results
1580                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1581                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1582
1583                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1584                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1585                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1586                         }
1587                         else if (isGcReliabilityFramework(scenario)) {
1588                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1589                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1590                         }
1591                         else {
1592                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1593                         }
1594                     }
1595
1596                     if (!enableCorefxTesting) {
1597                         // Run the rest of the build
1598                         // Build the mscorlib for the other OS's
1599                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1600                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1601
1602                         if (arch == "x64") {
1603                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1604                         }
1605
1606                         // Zip up the tests directory so that we don't use so much space/time copying
1607                         // 10s of thousands of files around.
1608                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1609
1610                         if (!isJitStressScenario(scenario)) {
1611                             // For windows, pull full test results and test drops for x86/x64.
1612                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1613                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1614                         }
1615
1616                         if (scenario == 'jitdiff') {
1617                             // retrive jit-dasm output for base commit, and run jit-diff
1618                             if (!isBuildOnly) {
1619                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1620                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1621                             }
1622                         }
1623
1624                         if (!isBuildOnly) {
1625                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1626                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1627                         }
1628                     }
1629                     break
1630                 case 'armlb':
1631                 case 'arm':
1632                     assert isArmWindowsScenario(scenario)
1633
1634                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1635                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1636
1637                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1638
1639                     def buildArchitecture = 'arm'
1640
1641                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1642                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1643                     def armCrossgenOpt = ''
1644                     if (architecture == 'armlb') {
1645                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1646                     }
1647
1648                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1649                     priority = '1'
1650
1651                     // This is now a build only job. Do not run tests. Use the flow job.
1652                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1653
1654                     // Zip up the tests directory so that we don't use so much space/time copying
1655                     // 10s of thousands of files around.
1656                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1657
1658                     // Add archival.
1659                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1660                     break
1661                 case 'arm64':
1662                     assert isArmWindowsScenario(scenario)
1663
1664                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1665                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1666
1667                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1668
1669                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1670                     priority = '1'
1671
1672                     // This is now a build only job. Do not run tests. Use the flow job.
1673                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1674
1675                     // Zip up the tests directory so that we don't use so much space/time copying
1676                     // 10s of thousands of files around.
1677                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1678
1679                     // Add archival.
1680                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1681                     break
1682                 default:
1683                     println("Unknown architecture: ${architecture}");
1684                     assert false
1685                     break
1686             }
1687             break
1688         // editor brace matching: }
1689         case 'Ubuntu':
1690         case 'Ubuntu16.04':
1691         case 'Ubuntu16.10':
1692         case 'Debian8.4':
1693         case 'OSX10.12':
1694         case 'CentOS7.1':
1695         case 'RHEL7.2':
1696         case 'Tizen':
1697         case 'Fedora24': // editor brace matching: {
1698             switch (architecture) {
1699                 case 'x64':
1700                 case 'x86':
1701                     if (architecture == 'x86' && os == 'Ubuntu') {
1702                         // build and PAL test
1703                         def dockerImage = getDockerImageName(architecture, os, true)
1704                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1705                         dockerImage = getDockerImageName(architecture, os, false)
1706                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1707                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1708                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1709                         break
1710                     }
1711
1712                     if (scenario == 'formatting') {
1713                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1714                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1715                         break
1716                     }
1717
1718                     if (scenario == 'illink') {
1719                         assert(os == 'Ubuntu')
1720                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1721                     }
1722
1723                     if (!enableCorefxTesting) {
1724                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1725                         // only on supported OS platforms.
1726                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1727                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1728                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1729                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1730
1731                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1732
1733                         // Basic archiving of the build
1734                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1735                         // And pal tests
1736                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1737                     }
1738                     else {
1739                         // Corefx stress testing
1740                         assert os == 'Ubuntu'
1741                         assert architecture == 'x64'
1742                         assert lowerConfiguration == 'checked'
1743                         assert isJitStressScenario(scenario)
1744
1745                         // Build coreclr
1746                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1747
1748                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1749                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1750
1751                         // Build and text corefx
1752                         def workspaceRelativeFxRoot = "_/fx"
1753                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1754
1755                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1756
1757                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1758
1759                         // Archive and process (only) the test results
1760                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1761                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1762                     }
1763                     break
1764                 case 'arm64':
1765                     if (!enableCorefxTesting) {
1766                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1767
1768                         // HACK -- Arm64 does not have corefx jobs yet.
1769                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1770                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1771                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1772                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1773
1774                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1775                         // Basic archiving of the build
1776                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1777                     }
1778                     break
1779                 case 'arm':
1780                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1781                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1782
1783                     // default values for Ubuntu
1784                     def arm_abi="arm"
1785                     def linuxCodeName="trusty"
1786                     if (os == 'Ubuntu16.04') {
1787                         linuxCodeName="xenial"
1788                     }
1789                     else if (os == 'Tizen') {
1790                         arm_abi="armel"
1791                         linuxCodeName="tizen"
1792                     }
1793
1794                     // Unzip the Windows test binaries first. Exit with 0
1795                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1796
1797                     // Unpack the corefx binaries
1798                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1799                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1800                     if (os != 'Tizen') {
1801                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1802                     }
1803                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1804                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1805                     if (os == 'Ubuntu') {
1806                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1807                     }
1808
1809                     // Call the ARM CI script to cross build and test using docker
1810                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1811                     --mode=docker \\
1812                     --${arm_abi} \\
1813                     --linuxCodeName=${linuxCodeName} \\
1814                     --buildConfig=${lowerConfiguration} \\
1815                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1816                     --coreFxBinDir=./bin/CoreFxBinDir \\
1817                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1818
1819                     // Basic archiving of the build, no pal tests
1820                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1821                     break
1822                 default:
1823                     println("Unknown architecture: ${architecture}");
1824                     assert false
1825                     break
1826             }
1827             break
1828         // editor brace matching: }
1829         default:
1830             println("Unknown os: ${os}");
1831             assert false
1832             break
1833     } // os
1834
1835     return buildCommands
1836 }
1837
1838 Constants.allScenarios.each { scenario ->
1839     [true, false].each { isPR ->
1840         Constants.architectureList.each { architecture ->
1841             Constants.configurationList.each { configuration ->
1842                 Constants.osList.each { os ->
1843                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1844                     // and reset the os to Windows_NT
1845                     def isBuildOnly = false
1846                     if (os == 'Windows_NT_BuildOnly') {
1847                         isBuildOnly = true
1848                         os = 'Windows_NT'
1849                     }
1850
1851                     // Tizen is only supported for arm architecture
1852                     if (os == 'Tizen' && architecture != 'arm') {
1853                         return
1854                     }
1855
1856                     // Skip totally unimplemented (in CI) configurations.
1857                     switch (architecture) {
1858                         case 'arm64':
1859                             if (os == 'Ubuntu16.04') {
1860                                 os = 'Ubuntu'
1861                             }
1862
1863                             // Windows and Ubuntu only
1864                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1865                                 return
1866                             }
1867                             break
1868                         case 'arm':
1869                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1870                                 return
1871                             }
1872                             break
1873                         case 'armlb':
1874                             if (os != 'Windows_NT') {
1875                                 return
1876                             }
1877                             break
1878                         case 'x86':
1879                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1880                                 return
1881                             }
1882                             break
1883                         case 'x86_arm_altjit':
1884                         case 'x64_arm64_altjit':
1885                             if (os != 'Windows_NT') {
1886                                 return
1887                             }
1888                             break
1889                         case 'x64':
1890                             // Everything implemented
1891                             break
1892                         default:
1893                             println("Unknown architecture: ${architecture}")
1894                             assert false
1895                             break
1896                     }
1897
1898                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1899                     // with checked builds)
1900                     if (isJitStressScenario(scenario)) {
1901                         if (configuration != 'Checked') {
1902                             return
1903                         }
1904
1905                         // Since these are just execution time differences,
1906                         // skip platforms that don't execute the tests here (Windows_NT only)
1907                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1908                         if (!isEnabledOS || isBuildOnly) {
1909                             return
1910                         }
1911
1912                         switch (architecture) {
1913                             case 'x64':
1914                             case 'x86':
1915                             case 'x86_arm_altjit':
1916                             case 'x64_arm64_altjit':
1917                                 // x86 ubuntu: default only
1918                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1919                                     return
1920                                 }
1921                                 // Windows: Everything implemented
1922                                 break
1923
1924                             default:
1925                                 // arm, arm64, armlb: stress is handled through flow jobs.
1926                                 return
1927                         }
1928                     }
1929                     else if (isR2RScenario(scenario)) {
1930                         if (os != 'Windows_NT') {
1931                             return
1932                         }
1933                         // R2R test runs are not implemented for arm/armlb/arm64.
1934                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1935                             return
1936                         }
1937                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1938                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1939                             return
1940                         }
1941                     }
1942                     else {
1943                         // Skip scenarios
1944                         switch (scenario) {
1945                             case 'ilrt':
1946                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1947                                 // the default scenario build
1948                                 if (os != 'Windows_NT') {
1949                                     return
1950                                 }
1951                                 // Only x64 for now
1952                                 if (architecture != 'x64') {
1953                                     return
1954                                 }
1955                                 // Release only
1956                                 if (configuration != 'Release') {
1957                                     return
1958                                 }
1959                                 break
1960                             case 'jitdiff':
1961                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1962                                     return
1963                                 }
1964                                 if (architecture != 'x64') {
1965                                     return
1966                                 }
1967                                 if (configuration != 'Checked') {
1968                                     return
1969                                 }
1970                                 break
1971                             case 'longgc':
1972                             case 'gcsimulator':
1973                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1974                                     return
1975                                 }
1976                                 if (architecture != 'x64') {
1977                                     return
1978                                 }
1979                                 if (configuration != 'Release') {
1980                                     return
1981                                 }
1982                                 break
1983                             case 'gc_reliability_framework':
1984                             case 'standalone_gc':
1985                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1986                                     return
1987                                 }
1988
1989                                 if (architecture != 'x64') {
1990                                     return
1991                                 }
1992
1993                                 if (configuration != 'Release' && configuration != 'Checked') {
1994                                     return
1995                                 }
1996                                 break
1997                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1998                             case 'formatting':
1999                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2000                                     return
2001                                 }
2002                                 if (architecture != 'x64') {
2003                                     return
2004                                 }
2005                                 if (configuration != 'Checked') {
2006                                     return
2007                                 }
2008                                 if (isBuildOnly) {
2009                                     return
2010                                 }
2011                                 break
2012                             case 'illink':
2013                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2014                                     return
2015                                 }
2016                                 if (architecture != 'x64' && architecture != 'x86') {
2017                                     return
2018                                 }
2019                                 if (isBuildOnly) {
2020                                     return
2021                                 }
2022                                 break
2023                             case 'default':
2024                                 // Nothing skipped
2025                                 break
2026                             default:
2027                                 println("Unknown scenario: ${scenario}")
2028                                 assert false
2029                                 break
2030                         }
2031                     }
2032
2033                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2034                     // runtime behavior, not compile-time behavior, are not interesting.
2035                     switch (architecture) {
2036                         case 'x86_arm_altjit':
2037                         case 'x64_arm64_altjit':
2038                             if (isGCStressRelatedTesting(scenario)) {
2039                                 return
2040                             }
2041                             break
2042                         default:
2043                             break
2044                     }
2045
2046                     // Calculate names
2047                     def lowerConfiguration = configuration.toLowerCase()
2048                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2049                     def folderName = getJobFolder(scenario)
2050
2051                     // Create the new job
2052                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2053
2054                     def machineAffinityOptions = null
2055
2056                     if (os != 'Windows_NT') {
2057                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2058                     }
2059                     else {
2060                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2061                     }
2062
2063                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2064
2065                     // Add all the standard options
2066                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2067                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2068
2069                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2070                     def osGroup = getOSGroup(os)
2071
2072                     newJob.with {
2073                         steps {
2074                             if (os == 'Windows_NT') {
2075                                 buildCommands.each { buildCommand ->
2076                                     batchFile(buildCommand)
2077                                 }
2078                             }
2079                             else {
2080                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2081                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2082                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2083                                     // Define the Windows Tests and Corefx build job names
2084                                     def WindowTestsName = projectFolder + '/' +
2085                                                           Utilities.getFullJobName(project,
2086                                                                                    getJobName(lowerConfiguration,
2087                                                                                               'x64' ,
2088                                                                                               'windows_nt',
2089                                                                                               'default',
2090                                                                                               true),
2091                                                                                    false)
2092                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2093                                                        Utilities.getFolderName(branch)
2094
2095                                     // Copy the Windows test binaries and the Corefx build binaries
2096                                     copyArtifacts(WindowTestsName) {
2097                                         includePatterns('bin/tests/tests.zip')
2098                                         buildSelector {
2099                                             latestSuccessful(true)
2100                                         }
2101                                     }
2102
2103                                     def arm_abi = 'arm'
2104                                     def corefx_os = 'linux'
2105                                     if (os == 'Tizen') {
2106                                         arm_abi = 'armel'
2107                                         corefx_os = 'tizen'
2108                                     }
2109
2110                                     // Let's use release CoreFX to test checked CoreCLR,
2111                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2112                                     def corefx_lowerConfiguration = lowerConfiguration
2113                                     if ( lowerConfiguration == 'checked' ) {
2114                                         corefx_lowerConfiguration='release'
2115                                     }
2116
2117                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2118                                         includePatterns('bin/build.tar.gz')
2119                                         buildSelector {
2120                                             latestSuccessful(true)
2121                                         }
2122                                     }
2123                                 }
2124
2125                                 buildCommands.each { buildCommand ->
2126                                     shell(buildCommand)
2127                                 }
2128                             }
2129                         }
2130                     } // newJob.with
2131
2132                 } // os
2133             } // configuration
2134         } // architecture
2135     } // isPR
2136 } // scenario
2137
2138
2139 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2140 Constants.allScenarios.each { scenario ->
2141     [true, false].each { isPR ->
2142         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2143             Constants.crossList.each { os ->
2144                 if (architecture == 'arm64') {
2145                     if (os != "Ubuntu" && os != "Windows_NT") {
2146                         return
2147                     }
2148                 } else if (architecture == 'arm' || architecture == 'armlb') {
2149                     if (os != 'Windows_NT') {
2150                         return
2151                     }
2152                 }
2153                 else if (architecture == 'x86') {
2154                     if (os != "Ubuntu") {
2155                         return
2156                     }
2157                 }
2158
2159                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2160
2161                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2162                     return
2163                 }
2164
2165                 Constants.configurationList.each { configuration ->
2166
2167                     // First, filter based on OS.
2168
2169                     if (os == 'Windows_NT') {
2170                         if (!isArmWindowsScenario(scenario)) {
2171                             return
2172                         }
2173                     }
2174                     else {
2175                         // Non-Windows
2176                         if (architecture == 'arm64') {
2177                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2178                                 return
2179                             }
2180                         }
2181                         else if (architecture == 'x86') {
2182                             // Linux/x86 only want default test
2183                             if (scenario != 'default') {
2184                                 return
2185                             }
2186                         }
2187                     }
2188
2189                     // For CentOS, we only want Checked/Release builds.
2190                     if (os == 'CentOS7.1') {
2191                         if (configuration != 'Checked' && configuration != 'Release') {
2192                             return
2193                         }
2194                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2195                             return
2196                         }
2197                     }
2198
2199                     // For RedHat and Debian, we only do Release builds.
2200                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2201                         if (configuration != 'Release') {
2202                             return
2203                         }
2204                         if (scenario != 'default') {
2205                             return
2206                         }
2207                     }
2208
2209                     // Next, filter based on scenario.
2210
2211                     if (isJitStressScenario(scenario)) {
2212                         if (configuration != 'Checked') {
2213                             return
2214                         }
2215                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2216                         if (isCoreFxScenario(scenario)) {
2217                             return
2218                         }
2219                     }
2220                     else if (isR2RBaselineScenario(scenario)) {
2221                         if (configuration != 'Checked' && configuration != 'Release') {
2222                             return
2223                         }
2224                     }
2225                     else if (isR2RStressScenario(scenario)) {
2226                         if (configuration != 'Checked') {
2227                             return
2228                         }
2229                     }
2230                     else {
2231                         // Skip scenarios
2232                         switch (scenario) {
2233                             case 'ilrt':
2234                             case 'longgc':
2235                             case 'gcsimulator':
2236                                 // Long GC tests take a long time on non-Release builds
2237                                 // ilrt is also Release only
2238                                 if (configuration != 'Release') {
2239                                     return
2240                                 }
2241                                 break
2242                             case 'jitdiff':
2243                                 if (configuration != 'Checked') {
2244                                     return;
2245                                 }
2246                                 break
2247                             case 'gc_reliability_framework':
2248                             case 'standalone_gc':
2249                                 if (configuration != 'Release' && configuration != 'Checked') {
2250                                     return
2251                                 }
2252                                 break
2253                             case 'formatting':
2254                                 return
2255                             case 'illink':
2256                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2257                                     return
2258                                 }
2259                                 break
2260                             case 'default':
2261                                 // Nothing skipped
2262                                 break
2263                             default:
2264                                 println("Unknown scenario: ${scenario}")
2265                                 assert false
2266                                 break
2267                         }
2268                     }
2269
2270                     // Done filtering. Now, create the jobs.
2271
2272                     def lowerConfiguration = configuration.toLowerCase()
2273                     def osGroup = getOSGroup(os)
2274                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2275
2276                     def inputCoreCLRBuildName = projectFolder + '/' +
2277                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2278
2279                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2280                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2281                     def testBuildScenario = scenario
2282                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2283                         testBuildScenario = 'default'
2284                     }
2285
2286                     def inputWindowsTestBuildArch = architecture
2287                     if (architecture == "arm64" && os != "Windows_NT") {
2288                         // Use the x64 test build for arm64 unix
2289                         inputWindowsTestBuildArch = "x64"
2290                     }
2291
2292                     def inputWindowTestsBuildName = projectFolder + '/' +
2293                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2294
2295                     // Enable Server GC for Ubuntu PR builds
2296                     def serverGCString = ''
2297                     if (os == 'Ubuntu' && isPR) {
2298                         serverGCString = '--useServerGC'
2299                     }
2300
2301                     def testOpts = ''
2302
2303                     if (isR2RScenario(scenario)) {
2304
2305                         testOpts += ' --crossgen --runcrossgentests'
2306
2307                         if (scenario == 'r2r_jitstress1') {
2308                             testOpts += ' --jitstress=1'
2309                         }
2310                         else if (scenario == 'r2r_jitstress2') {
2311                             testOpts += ' --jitstress=2'
2312                         }
2313                         else if (scenario == 'r2r_jitstressregs1') {
2314                             testOpts += ' --jitstressregs=1'
2315                         }
2316                         else if (scenario == 'r2r_jitstressregs2') {
2317                             testOpts += ' --jitstressregs=2'
2318                         }
2319                         else if (scenario == 'r2r_jitstressregs3') {
2320                             testOpts += ' --jitstressregs=3'
2321                         }
2322                         else if (scenario == 'r2r_jitstressregs4') {
2323                             testOpts += ' --jitstressregs=4'
2324                         }
2325                         else if (scenario == 'r2r_jitstressregs8') {
2326                             testOpts += ' --jitstressregs=8'
2327                         }
2328                         else if (scenario == 'r2r_jitstressregs0x10') {
2329                             testOpts += ' --jitstressregs=0x10'
2330                         }
2331                         else if (scenario == 'r2r_jitstressregs0x80') {
2332                             testOpts += ' --jitstressregs=0x80'
2333                         }
2334                         else if (scenario == 'r2r_jitstressregs0x1000') {
2335                             testOpts += ' --jitstressregs=0x1000'
2336                         }
2337                         else if (scenario == 'r2r_jitminopts') {
2338                             testOpts += ' --jitminopts'
2339                         }
2340                         else if (scenario == 'r2r_jitforcerelocs') {
2341                             testOpts += ' --jitforcerelocs'
2342                         }
2343                         else if (scenario == 'r2r_gcstress15') {
2344                             testOpts += ' --gcstresslevel=0xF'
2345                         }
2346                     }
2347                     else if (scenario == 'jitdiff') {
2348                         testOpts += ' --jitdisasm --crossgen'
2349                     }
2350                     else if (scenario == 'illink') {
2351                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2352                     }
2353                     else if (isLongGc(scenario)) {
2354                         // Long GC tests behave very poorly when they are not
2355                         // the only test running (many of them allocate until OOM).
2356                         testOpts += ' --sequential'
2357
2358                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2359                         // for running long GC and GCSimulator tests, respectively. We don't use them
2360                         // here because using a playlist file produces much more readable output on the CI machines
2361                         // and reduces running time.
2362                         //
2363                         // The Long GC playlist contains all of the tests that are
2364                         // going to be run. The GCSimulator playlist contains all of
2365                         // the GC simulator tests.
2366                         if (scenario == 'longgc') {
2367                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2368                         }
2369                         else if (scenario == 'gcsimulator') {
2370                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2371                         }
2372                     }
2373                     else if (isGcReliabilityFramework(scenario)) {
2374                         testOpts += ' --build-overlay-only'
2375                     }
2376                     else if (scenario == 'standalone_gc') {
2377                         if (osGroup == 'OSX') {
2378                             testOpts += ' --gcname=libclrgc.dylib'
2379                         }
2380                         else if (osGroup == 'Linux') {
2381                             testOpts += ' --gcname=libclrgc.so'
2382                         }
2383                         else {
2384                             println("Unexpected OS group: ${osGroup} for os ${os}")
2385                             assert false
2386                         }
2387                     }
2388
2389                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2390
2391                     def folder = getJobFolder(scenario)
2392                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2393                         // Add parameters for the inputs
2394
2395                         if (windowsArmJob == true) {
2396                             parameters {
2397                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2398                             }
2399                         }
2400                         else {
2401                             parameters {
2402                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2403                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2404                             }
2405                         }
2406
2407                         steps {
2408                             // Set up the copies
2409
2410                             // Coreclr build containing the tests and mscorlib
2411
2412                             if (windowsArmJob != true) {
2413                                 copyArtifacts(inputWindowTestsBuildName) {
2414                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2415                                     buildSelector {
2416                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2417                                     }
2418                                 }
2419                             }
2420
2421                             // Coreclr build we are trying to test
2422
2423                             copyArtifacts(inputCoreCLRBuildName) {
2424                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2425                                 buildSelector {
2426                                     buildNumber('${CORECLR_BUILD}')
2427                                 }
2428                             }
2429
2430                             // Windows CoreCLR Arm(64) will restore corefx
2431                             // packages correctly.
2432                             //
2433                             // In addition, test steps are entirely different
2434                             // because we do not have a unified runner
2435                             if (windowsArmJob != true) {
2436                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2437
2438                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2439                                 def osJobName
2440                                 if (os == 'Ubuntu') {
2441                                     osJobName = 'ubuntu14.04'
2442                                 }
2443                                 else if (architecture == 'x86') {
2444                                     if (os == 'Ubuntu') {
2445                                         // Linux/x86 corefx jobs does not build managed yet
2446                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2447                                         osJobName = "linux_arm_cross"
2448                                     }
2449                                 }
2450                                 else {
2451                                     osJobName = os.toLowerCase()
2452                                 }
2453                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2454                                     includePatterns('bin/build.tar.gz')
2455                                     buildSelector {
2456                                         latestSuccessful(true)
2457                                     }
2458                                 }
2459
2460                                 shell("mkdir ./bin/CoreFxBinDir")
2461                                 // Unpack the corefx binaries
2462                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2463
2464                                 // HACK -- Arm64 does not have corefx jobs yet.
2465                                 // Clone corefx and build the native packages overwriting the x64 packages.
2466                                 if (architecture == 'arm64') {
2467                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2468                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2469                                 }
2470                                 else if (architecture == 'x86') {
2471                                     shell("mkdir ./bin/CoreFxNative")
2472
2473                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2474                                         includePatterns('bin/build.tar.gz')
2475                                         targetDirectory('bin/CoreFxNative')
2476                                         buildSelector {
2477                                             latestSuccessful(true)
2478                                         }
2479                                     }
2480
2481                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2482                                 }
2483
2484                                 // Unzip the tests first.  Exit with 0
2485                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2486
2487                                 // Execute the tests
2488                                 def runDocker = isNeedDocker(architecture, os, false)
2489                                 def dockerPrefix = ""
2490                                 def dockerCmd = ""
2491                                 if (runDocker) {
2492                                     def dockerImage = getDockerImageName(architecture, os, false)
2493                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2494                                     dockerCmd = dockerPrefix + "${dockerImage} "
2495                                 }
2496
2497                                 // If we are running a stress mode, we'll set those variables first
2498                                 def testEnvOpt = ""
2499                                 if (isJitStressScenario(scenario)) {
2500                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2501                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2502                                     shell("${createScriptCmds}")
2503                                     testEnvOpt = "--test-env=" + scriptFileName
2504                                 }
2505
2506                                 if (isGCStressRelatedTesting(scenario)) {
2507                                     shell('./init-tools.sh')
2508                                 }
2509
2510                                 shell("""${dockerCmd}./tests/runtest.sh \\
2511                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2512                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2513                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2514                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2515                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2516                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2517
2518                                 if (isGcReliabilityFramework(scenario)) {
2519                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2520                                     if (serverGCString != '') {
2521                                         if (runDocker) {
2522                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2523                                         }
2524                                         else {
2525                                             shell("export COMPlus_gcServer=1")
2526                                         }
2527                                     }
2528
2529                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2530                                 }
2531                             }
2532
2533                             else { // windowsArmJob == true
2534                                 // Unzip tests.
2535                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2536
2537                                 // Build the build commands
2538                                 def buildCommands = ""
2539
2540                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2541                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2542                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2543
2544                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2545                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2546
2547                                 // For all jobs
2548                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2549
2550                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2551                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2552
2553                                 // ARM legacy backend; this is an altjit.
2554                                 if (architecture == "armlb") {
2555                                     addEnvVariable("COMPlus_AltJit", "*")
2556                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2557                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2558                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2559                                 }
2560
2561                                 // If we are running a stress mode, we'll set those variables as well
2562                                 def stressValues = null
2563                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2564                                     if (isJitStressScenario(scenario)) {
2565                                         stressValues = Constants.jitStressModeScenarios[scenario]
2566                                     }
2567                                     else {
2568                                         stressValues = Constants.r2rStressScenarios[scenario]
2569                                     }
2570
2571                                     stressValues.each { key, value ->
2572                                         addEnvVariable(key, value)
2573                                     }
2574                                 }
2575
2576                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2577                                 // crossgen the framework assemblies, etc.
2578
2579                                 // Create the smarty command
2580                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2581                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2582                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2583
2584                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2585
2586                                 if (architecture == "armlb") {
2587                                     addExclude("LEGACYJIT_FAIL")
2588                                 }
2589
2590                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2591                                     def failTag = "JITSTRESS_FAIL"
2592                                     def excludeTag = "JITSTRESS_EXCLUDE"
2593
2594                                     if (scenario.contains('gc')) {
2595                                         failTag = "GCSTRESS_FAIL"
2596                                         excludeTag = "GCSTRESS_EXCLUDE"
2597                                     }
2598
2599                                     addArchSpecificExclude(architecture, failTag)
2600                                     addArchSpecificExclude(architecture, excludeTag)
2601                                 }
2602                                 else {
2603                                     addExclude("pri1")
2604                                 }
2605
2606                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2607                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2608                                 addExclude("LONG_RUNNING")
2609
2610                                 smartyCommand += "/lstFile Tests.lst"
2611
2612                                 def testListArch = [
2613                                     'arm64': 'arm64',
2614                                     'arm': 'arm',
2615                                     'armlb': 'arm'
2616                                 ]
2617
2618                                 def archLocation = testListArch[architecture]
2619
2620                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2621                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2622                                 addCommand("${smartyCommand}")
2623
2624                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2625                                 // However, we also need to remove all the variables that were set during this batch file, so we
2626                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2627                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2628                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2629                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2630                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2631                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2632
2633                                 // ZIP up the smarty output, no matter what the smarty result.
2634                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2635
2636                                 addCommand("echo %errorlevel%")
2637                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2638
2639                                 // Use the smarty errorlevel as the script errorlevel.
2640                                 addCommand("exit /b %__save_smarty_errorlevel%")
2641
2642                                 batchFile(buildCommands)
2643                             }
2644                         }
2645                     }
2646
2647                     if (scenario == 'jitdiff') {
2648                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2649                     }
2650
2651                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2652                     if (os in ['Ubuntu']) {
2653                         SummaryBuilder summaries = new SummaryBuilder()
2654                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2655                         summaries.emit(newJob)
2656                     }
2657
2658                     def affinityOptions = null
2659
2660                     if (windowsArmJob == true) {
2661                         affinityOptions = [
2662                             "use_arm64_build_machine" : false
2663                         ]
2664                     }
2665
2666                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2667                         affinityOptions = [
2668                             "large_pages" : false
2669                         ]
2670                     }
2671
2672                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2673                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2674
2675                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
2676
2677                     if (windowsArmJob != true) {
2678                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2679                     }
2680                     else {
2681                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2682
2683                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2684                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2685                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2686                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2687                     }
2688
2689                     // Create a build flow to join together the build and tests required to run this test.
2690                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2691                     // Linux CoreCLR test
2692                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2693                     def fullTestJobName = projectFolder + '/' + newJob.name
2694                     // Add a reference to the input jobs for report purposes
2695                     JobReport.Report.addReference(inputCoreCLRBuildName)
2696                     JobReport.Report.addReference(inputWindowTestsBuildName)
2697                     JobReport.Report.addReference(fullTestJobName)
2698                     def newFlowJob = null
2699
2700                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2701                         // Do not create the flow job for RHEL jobs.
2702                         return
2703                     }
2704
2705                     if (windowsArmJob == true) {
2706                         // For Windows arm jobs there is no reason to build a parallel test job.
2707                         // The product build supports building and archiving the tests.
2708
2709                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2710                         buildFlow("""
2711 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2712
2713 // And then build the test build
2714 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2715 """)
2716                         }
2717                     }
2718                     else {
2719                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2720                         buildFlow("""
2721 // Build the input jobs in parallel
2722 parallel (
2723 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2724 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2725 )
2726
2727 // And then build the test build
2728 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2729             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2730 """)
2731                         }
2732                     }
2733
2734                     // For the flow jobs set the machine affinity as x64 if an armarch.
2735                     def flowArch = architecture
2736
2737                     if (flowArch in validWindowsNTCrossArches) {
2738                         flowArch = 'x64'
2739                         affinityOptions = null
2740                     }
2741
2742                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2743                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2744                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2745                 } // configuration
2746             } // os
2747         } // architecture
2748     } // isPR
2749 } // scenario
2750
2751 JobReport.Report.generateJobReport(out)
2752
2753 // Make the call to generate the help job
2754 Utilities.createHelperJob(this, project, branch,
2755     "Welcome to the ${project} Repository",  // This is prepended to the help message
2756     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2757
2758 Utilities.addCROSSCheck(this, project, branch)