netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 // **************************************************************
 258 // Create some specific views
 259 //
 260 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 261 // views based on a single regular expression. These views will be generated by adding a
 262 // specific set of jobs to them.
 263 //
 264 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 265 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 266 // those are required, don't add them (which simplifies the view pages, as well).
 267 // **************************************************************
 268
 269 // MergeJobView: include all jobs that execute when a PR change is merged.
 270 def static MergeJobView = listView('Merge') {
 271     columns {
 272         status()
 273         weather()
 274         name()
 275         lastSuccess()
 276         lastFailure()
 277         lastDuration()
 278         buildButton()
 279     }
 280 }
 281
 282 // PeriodicJobView: include all jobs that execute on a schedule
 283 def static PeriodicJobView = listView('Periodic') {
 284     columns {
 285         status()
 286         weather()
 287         name()
 288         lastSuccess()
 289         lastFailure()
 290         lastDuration()
 291         buildButton()
 292     }
 293 }
 294
 295 // Create a view for non-PR jobs for each architecture.
 296 def static ArchitectureViews = [:]
 297 Constants.architectureList.each { architecture ->
 298     ArchitectureViews[architecture] = listView(architecture) {
 299         columns {
 300             status()
 301             weather()
 302             name()
 303             lastSuccess()
 304             lastFailure()
 305             lastDuration()
 306             buildButton()
 307         }
 308     }
 309 }
 310
 311 // Create a view for non-PR jobs for each OS.
 312 def static OSViews = [:]
 313 Constants.osList.each { os ->
 314     // Don't create one for the special 'Windows_NT_BuildOnly'
 315     if (os == 'Windows_NT_BuildOnly') {
 316         return
 317     }
 318     OSViews[os] = listView(os) {
 319         columns {
 320             status()
 321             weather()
 322             name()
 323             lastSuccess()
 324             lastFailure()
 325             lastDuration()
 326             buildButton()
 327         }
 328     }
 329 }
 330
 331 def static addToMergeView(def job) {
 332     MergeJobView.with {
 333         jobs {
 334             name(job.name)
 335         }
 336     }
 337 }
 338
 339 def static addToPeriodicView(def job) {
 340     PeriodicJobView.with {
 341         jobs {
 342             name(job.name)
 343         }
 344     }
 345 }
 346
 347 def static addToViews(def job, def isPR, def architecture, def os) {
 348     if (isPR) {
 349         // No views want PR jobs currently.
 350         return
 351     }
 352
 353     // Add to architecture view.
 354     ArchitectureViews[architecture].with {
 355         jobs {
 356             name(job.name)
 357         }
 358     }
 359
 360     // Add to OS view.
 361     OSViews[os].with {
 362         jobs {
 363             name(job.name)
 364         }
 365     }
 366 }
 367
 368 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 369     addToPeriodicView(job)
 370     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 371 }
 372
 373 def static addGithubPushTriggerHelper(def job) {
 374     addToMergeView(job)
 375     Utilities.addGithubPushTrigger(job)
 376 }
 377
 378
 379 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 380     assert os instanceof String
 381     assert architecture instanceof String
 382
 383     def armArches = ['arm', 'armlb', 'arm64']
 384     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 385
 386     if (!(architecture in armArches)) {
 387         assert options == null
 388         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 389
 390         return
 391     }
 392
 393     // This is an arm(64) job.
 394     //
 395     // There are several options.
 396     //
 397     // Windows_NT
 398     //
 399     // Arm32 (Build) -> latest-arm64
 400     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 401     // Arm32 (Test)  -> arm64-windows_nt
 402     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 403     //
 404     // Arm64 (Build) -> latest-arm64
 405     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 406     // Arm64 (Test)  -> arm64-windows_nt
 407     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 408     //
 409     // Ubuntu
 410     //
 411     // Arm32 (Build) -> arm-cross-latest
 412     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 413     // Arm32 (Test)  -> NYI Arch not supported
 414     //       |->
 415     //
 416     // Arm64 (Build) -> arm64-cross-latest
 417     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 418     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 419     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 420     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 421     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 422
 423     // This has to be a arm arch
 424     assert architecture in armArches
 425     if (os == "Windows_NT") {
 426         // Arm(64) Windows jobs share the same machines for now
 427         def isBuild = options['use_arm64_build_machine'] == true
 428
 429         if (isBuild == true) {
 430             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 431         } else {
 432             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 433         }
 434     } else {
 435         assert os != 'Windows_NT'
 436         assert os in supportedArmLinuxOs
 437
 438         if (architecture == 'arm' || architecture == 'armlb') {
 439             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 440         } else {
 441             // Arm64 Linux
 442             if (options['is_build_only'] == true) {
 443                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 444             } else {
 445                 // Arm64 Test Machines
 446                 if (options['large_pages'] == false) {
 447                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 448                 } else {
 449                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 450                 }
 451             }
 452         }
 453     }
 454 }
 455
 456 def static isGCStressRelatedTesting(def scenario) {
 457     // The 'r2r_gcstress15' scenario is a basic scenario.
 458     // Detect it and make it a GCStress related.
 459     if (scenario == 'r2r_gcstress15')
 460     {
 461         return true;
 462     }
 463
 464     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 465     def scenarioName = scenario.toLowerCase()
 466     def isGCStressTesting = false
 467     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 468         if (k in gcStressTestEnvVars) {
 469             isGCStressTesting = true;
 470         }
 471     }
 472     return isGCStressTesting
 473 }
 474
 475 def static isCoreFxScenario(def scenario) {
 476     def corefx_prefix = 'corefx_'
 477     if (scenario.length() < corefx_prefix.length()) {
 478         return false
 479     }
 480     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 481 }
 482
 483 def static isR2RBaselineScenario(def scenario) {
 484     return (scenario == 'r2r')
 485 }
 486
 487 def static isR2RStressScenario(def scenario) {
 488     return Constants.r2rStressScenarios.containsKey(scenario)
 489 }
 490
 491 def static isR2RScenario(def scenario) {
 492     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 493 }
 494
 495 def static isJitStressScenario(def scenario) {
 496     return Constants.jitStressModeScenarios.containsKey(scenario)
 497 }
 498
 499 def static isLongGc(def scenario) {
 500     return (scenario == 'longgc' || scenario == 'gcsimulator')
 501 }
 502
 503 def static isJitDiff(def scenario) {
 504     return (scenario == 'jitdiff')
 505 }
 506
 507 def static isGcReliabilityFramework(def scenario) {
 508     return (scenario == 'gc_reliability_framework')
 509 }
 510
 511 def static isArmWindowsScenario(def scenario) {
 512     return Constants.validArmWindowsScenarios.containsKey(scenario)
 513 }
 514
 515 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 516     // 2 hours (120 minutes) is the default timeout
 517     def timeout = 120
 518
 519     if (!(scenario == 'default' && isPR == true)) {
 520         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 521         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 522         timeout = 240
 523     }
 524
 525     if (!isBuildOnly) {
 526         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 527         if (isGCStressRelatedTesting(scenario)) {
 528             timeout = 4320
 529         }
 530         else if (isCoreFxScenario(scenario)) {
 531             timeout = 360
 532         }
 533         else if (isJitStressScenario(scenario)) {
 534             timeout = 240
 535         }
 536         else if (isR2RBaselineScenario(scenario)) {
 537             timeout = 240
 538         }
 539         else if (isLongGc(scenario)) {
 540             timeout = 1440
 541         }
 542         else if (isJitDiff(scenario)) {
 543             timeout = 240
 544         }
 545         else if (isGcReliabilityFramework(scenario)) {
 546             timeout = 1440
 547         }
 548         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 549             timeout = 240
 550         }
 551     }
 552
 553     if (configuration == 'Debug') {
 554         // Debug runs can be very slow. Add an hour.
 555         timeout += 60
 556     }
 557
 558     // If we've changed the timeout from the default, set it in the job.
 559
 560     if (timeout != 120) {
 561         Utilities.setJobTimeout(newJob, timeout)
 562     }
 563 }
 564
 565 def static getJobFolder(def scenario) {
 566     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 567         return 'jitstress'
 568     }
 569     if (scenario == 'illink') {
 570         return 'illink'
 571     }
 572     return ''
 573 }
 574
 575 def static getStressModeDisplayName(def scenario) {
 576     def displayStr = ''
 577     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 578         def prefixLength = 'COMPlus_'.length()
 579         if (k.length() >= prefixLength) {
 580             def modeName = k.substring(prefixLength, k.length())
 581             displayStr += ' ' + modeName + '=' + v
 582         }
 583     }
 584
 585     if (isCoreFxScenario(scenario)) {
 586         displayStr = ('CoreFx ' + displayStr).trim()
 587     }
 588
 589     return displayStr
 590 }
 591
 592 def static getR2RDisplayName(def scenario) {
 593     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 594     def displayStr = scenario
 595     def prefixLength = 'r2r_'.length()
 596     if (displayStr.length() >= prefixLength) {
 597         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 598     } else if (scenario == 'r2r') {
 599         displayStr = "R2R"
 600     }
 601     return displayStr
 602 }
 603
 604 //
 605 // Functions to create an environment script.
 606 //      envScriptCreate -- initialize the script (call first)
 607 //      envScriptFinalize -- finalize the script (call last)
 608 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 609 //      envScriptAppendExistingScript -- append an existing script to the generated script
 610 //
 611 // Each script returns a string of commands. Concatenate all the strings together before
 612 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 613 //
 614
 615 // Initialize the environment setting script.
 616 def static envScriptCreate(def os, def stepScriptLocation) {
 617     def stepScript = ''
 618     if (os == 'Windows_NT') {
 619         stepScript += "echo Creating TestEnv script\r\n"
 620         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 621
 622         // Create at least an empty script.
 623         stepScript += "echo. > ${stepScriptLocation}\r\n"
 624     }
 625     else {
 626         stepScript += "echo Creating environment setting script\n"
 627         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 628     }
 629
 630     return stepScript
 631 }
 632
 633 // Generates the string for setting stress mode variables.
 634 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 635     def stepScript = ''
 636     if (os == 'Windows_NT') {
 637         stressModeVars.each{ k, v ->
 638             // Write out what we are writing to the script file
 639             stepScript += "echo Setting ${k}=${v}\r\n"
 640             // Write out the set itself to the script file`
 641             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 642         }
 643     }
 644     else {
 645         stressModeVars.each{ k, v ->
 646             // Write out what we are writing to the script file
 647             stepScript += "echo Setting ${k}=${v}\n"
 648             // Write out the set itself to the script file`
 649             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 650         }
 651     }
 652
 653     return stepScript
 654 }
 655
 656 // Append an existing script to an environment script.
 657 // Returns string of commands to do this.
 658 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 659     assert (os == 'Windows_NT')
 660     def stepScript = ''
 661
 662     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 663     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 664
 665     return stepScript
 666 }
 667
 668 // Finalize an environment setting script.
 669 // Returns string of commands to do this.
 670 def static envScriptFinalize(def os, def stepScriptLocation) {
 671     def stepScript = ''
 672
 673     if (os == 'Windows_NT') {
 674         // Display the resulting script. This is useful when looking at the output log file.
 675         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 676         stepScript += "type ${stepScriptLocation}\r\n"
 677     }
 678     else {
 679         stepScript += "chmod +x ${stepScriptLocation}\n"
 680     }
 681
 682     return stepScript
 683 }
 684
 685 def static isNeedDocker(def architecture, def os, def isBuild) {
 686     if (isBuild) {
 687         if (architecture == 'x86' && os == 'Ubuntu') {
 688             return true
 689         }
 690         else if (architecture == 'arm') {
 691             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 692                 return true
 693             }
 694         }
 695     }
 696     else {
 697         if (architecture == 'x86' && os == 'Ubuntu') {
 698             return true
 699         }
 700     }
 701     return false
 702 }
 703
 704 def static getDockerImageName(def architecture, def os, def isBuild) {
 705     // We must change some docker private images to official later
 706     if (isBuild) {
 707         if (architecture == 'x86' && os == 'Ubuntu') {
 708             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 709         }
 710         else if (architecture == 'arm') {
 711             if (os == 'Ubuntu') {
 712                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 713             }
 714             else if (os == 'Ubuntu16.04') {
 715                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 716             }
 717             else if (os == 'Tizen') {
 718                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 719             }
 720         }
 721     }
 722     else {
 723         if (architecture == 'x86' && os == 'Ubuntu') {
 724             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 725         }
 726     }
 727     println("Unknown architecture to use docker: ${architecture} ${os}");
 728     assert false
 729 }
 730
 731 // Calculates the name of the build job based on some typical parameters.
 732 //
 733 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 734     // If the architecture is x64, do not add that info into the build name.
 735     // Need to change around some systems and other builds to pick up the right builds
 736     // to do that.
 737
 738     def suffix = scenario != 'default' ? "_${scenario}" : '';
 739     if (isBuildOnly) {
 740         suffix += '_bld'
 741     }
 742     def baseName = ''
 743     switch (architecture) {
 744         case 'x64':
 745             if (scenario == 'default') {
 746                 // For now we leave x64 off of the name for compatibility with other jobs
 747                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 748             }
 749             else if (scenario == 'formatting') {
 750                 // we don't care about the configuration for the formatting job. It runs all configs
 751                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 752             }
 753             else {
 754                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 755             }
 756             break
 757         case 'arm64':
 758             if (os.toLowerCase() == "windows_nt") {
 759                 // These are cross builds
 760                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 761             }
 762             else {
 763                 // Defaults to a small page size set of machines.
 764                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 765             }
 766             break
 767         case 'arm':
 768             // These are cross builds
 769             if (os == 'Tizen') {
 770                 // ABI: softfp
 771                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 772             }
 773             else {
 774                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 775             }
 776             break
 777         case 'armlb':
 778             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 779             break
 780         case 'x86':
 781         case 'x86_arm_altjit':
 782         case 'x64_arm64_altjit':
 783             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 784             break
 785         default:
 786             println("Unknown architecture: ${architecture}");
 787             assert false
 788             break
 789     }
 790
 791     return baseName + suffix
 792 }
 793
 794 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 795
 796     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 797     if (os == 'Windows_NT') {
 798         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 799             if (!(branch in Constants.WindowsArm64Branches)) {
 800                 return
 801             }
 802         }
 803     }
 804
 805     // Check scenario.
 806     switch (scenario) {
 807         case 'default':
 808             switch (architecture) {
 809                 case 'x64':
 810                 case 'x86':
 811                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 812                         addPeriodicTriggerHelper(job, '@daily')
 813                     }
 814                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 815                         addGithubPushTriggerHelper(job)
 816                     }
 817                     break
 818                 case 'arm':
 819                 case 'armlb':
 820                 case 'x86_arm_altjit':
 821                 case 'x64_arm64_altjit':
 822                     addGithubPushTriggerHelper(job)
 823                     break
 824                 case 'arm64':
 825                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 826                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 827                     break
 828                 default:
 829                     println("Unknown architecture: ${architecture}");
 830                     assert false
 831                     break
 832             }
 833             break
 834         case 'r2r':
 835             assert !(os in bidailyCrossList)
 836             // r2r gets a push trigger for checked/release
 837             if (configuration == 'Checked' || configuration == 'Release') {
 838                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 839                 if (architecture == 'x64' && os != 'OSX10.12') {
 840                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 841                     if (isFlowJob || os == 'Windows_NT') {
 842                         addGithubPushTriggerHelper(job)
 843                     }
 844                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 845                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 846                     if (isFlowJob) {
 847                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 848                     }
 849                 }
 850                 // For x86, only add per-commit jobs for Windows
 851                 else if (architecture == 'x86') {
 852                     if (os == 'Windows_NT') {
 853                         addGithubPushTriggerHelper(job)
 854                     }
 855                 }
 856                 // arm64 r2r jobs should only run daily.
 857                 else if (architecture == 'arm64') {
 858                     if (os == 'Windows_NT') {
 859                         addPeriodicTriggerHelper(job, '@daily')
 860                     }
 861                 }
 862             }
 863             break
 864         case 'r2r_jitstress1':
 865         case 'r2r_jitstress2':
 866         case 'r2r_jitstressregs1':
 867         case 'r2r_jitstressregs2':
 868         case 'r2r_jitstressregs3':
 869         case 'r2r_jitstressregs4':
 870         case 'r2r_jitstressregs8':
 871         case 'r2r_jitstressregs0x10':
 872         case 'r2r_jitstressregs0x80':
 873         case 'r2r_jitstressregs0x1000':
 874         case 'r2r_jitminopts':
 875         case 'r2r_jitforcerelocs':
 876         case 'r2r_gcstress15':
 877             assert !(os in bidailyCrossList)
 878
 879             // GCStress=C is currently not supported on OS X
 880             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 881                 break
 882             }
 883
 884             // GC Stress 15 r2r gets a push trigger for checked/release
 885             if (configuration == 'Checked' || configuration == 'Release') {
 886                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 887                 if (architecture == 'x64') {
 888                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 889                     if (isFlowJob || os == 'Windows_NT') {
 890                         // Add a weekly periodic trigger
 891                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 892                     }
 893                 }
 894                 // For x86, only add per-commit jobs for Windows
 895                 else if (architecture == 'x86') {
 896                     if (os == 'Windows_NT') {
 897                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 898                     }
 899                 }
 900             }
 901             break
 902         case 'longgc':
 903             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 904             assert configuration == 'Release'
 905             assert architecture == 'x64'
 906             addPeriodicTriggerHelper(job, '@daily')
 907             // TODO: Add once external email sending is available again
 908             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 909             break
 910         case 'gcsimulator':
 911             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 912             assert configuration == 'Release'
 913             assert architecture == 'x64'
 914             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 915             // TODO: Add once external email sending is available again
 916             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 917             break
 918         case 'standalone_gc':
 919             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 920             assert (configuration == 'Release' || configuration == 'Checked')
 921             // TODO: Add once external email sending is available again
 922             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 923             addPeriodicTriggerHelper(job, '@daily')
 924             break
 925         case 'gc_reliability_framework':
 926             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 927             assert (configuration == 'Release' || configuration == 'Checked')
 928             // Only triggered by phrase.
 929             break
 930         case 'ilrt':
 931             assert !(os in bidailyCrossList)
 932             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 933             if (architecture == 'x64' && configuration == 'Release') {
 934                 // We don't expect to see a job generated except in these scenarios
 935                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 936                 if (isFlowJob || os == 'Windows_NT') {
 937                     addPeriodicTriggerHelper(job, '@daily')
 938                 }
 939             }
 940             break
 941         case 'jitdiff':
 942             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 943             assert configuration == 'Checked'
 944             assert (architecture == 'x64' || architecture == 'x86')
 945             addGithubPushTriggerHelper(job)
 946             break
 947         case 'formatting':
 948             assert (os == 'Windows_NT' || os == "Ubuntu")
 949             assert architecture == 'x64'
 950             addGithubPushTriggerHelper(job)
 951             break
 952         case 'jitstressregs1':
 953         case 'jitstressregs2':
 954         case 'jitstressregs3':
 955         case 'jitstressregs4':
 956         case 'jitstressregs8':
 957         case 'jitstressregs0x10':
 958         case 'jitstressregs0x80':
 959         case 'jitstressregs0x1000':
 960         case 'minopts':
 961         case 'forcerelocs':
 962         case 'jitstress1':
 963         case 'jitstress2':
 964         case 'jitstress2_jitstressregs1':
 965         case 'jitstress2_jitstressregs2':
 966         case 'jitstress2_jitstressregs3':
 967         case 'jitstress2_jitstressregs4':
 968         case 'jitstress2_jitstressregs8':
 969         case 'jitstress2_jitstressregs0x10':
 970         case 'jitstress2_jitstressregs0x80':
 971         case 'jitstress2_jitstressregs0x1000':
 972         case 'tailcallstress':
 973         case 'jitsse2only':
 974         case 'jitnosimd':
 975         case 'corefx_baseline':
 976         case 'corefx_minopts':
 977         case 'corefx_jitstress1':
 978         case 'corefx_jitstress2':
 979         case 'corefx_jitstressregs1':
 980         case 'corefx_jitstressregs2':
 981         case 'corefx_jitstressregs3':
 982         case 'corefx_jitstressregs4':
 983         case 'corefx_jitstressregs8':
 984         case 'corefx_jitstressregs0x10':
 985         case 'corefx_jitstressregs0x80':
 986         case 'corefx_jitstressregs0x1000':
 987         case 'zapdisable':
 988             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 989                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 990                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 991                     if (os == 'Windows_NT') {
 992                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 993                         addPeriodicTriggerHelper(job, '@weekly')
 994                     }
 995                 }
 996                 else {
 997                     addPeriodicTriggerHelper(job, '@daily')
 998                 }
 999             }
1000             break
1001         case 'heapverify1':
1002         case 'gcstress0x3':
1003             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1004                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1005                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1006                     if (os == 'Windows_NT') {
1007                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1008                         addPeriodicTriggerHelper(job, '@weekly')
1009                     }
1010                     // TODO: Add once external email sending is available again
1011                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1012                 }
1013                 else {
1014                     addPeriodicTriggerHelper(job, '@weekly')
1015                 }
1016             }
1017             break
1018         case 'gcstress0xc':
1019         case 'gcstress0xc_zapdisable':
1020         case 'gcstress0xc_zapdisable_jitstress2':
1021         case 'gcstress0xc_zapdisable_heapverify1':
1022         case 'gcstress0xc_jitstress1':
1023         case 'gcstress0xc_jitstress2':
1024         case 'gcstress0xc_minopts_heapverify1':
1025             // GCStress=C is currently not supported on OS X
1026             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1027                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1028                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1029                     if (os == 'Windows_NT') {
1030                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1031                         addPeriodicTriggerHelper(job, '@weekly')
1032                     }
1033                     // TODO: Add once external email sending is available again
1034                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1035                 }
1036                 else {
1037                     addPeriodicTriggerHelper(job, '@weekly')
1038                 }
1039             }
1040             break
1041
1042         case 'illink':
1043             // Testing on other operating systems TBD
1044             assert (os == 'Windows_NT' || os == 'Ubuntu')
1045             if (architecture == 'x64' || architecture == 'x86') {
1046                 if (configuration == 'Checked') {
1047                     addPeriodicTriggerHelper(job, '@daily')
1048                 }
1049             }
1050             break
1051
1052         case 'tieredcompilation':
1053         case 'corefx_tieredcompilation':
1054             // No periodic jobs just yet, still testing
1055             break
1056
1057         default:
1058             println("Unknown scenario: ${scenario}");
1059             assert false
1060             break
1061     }
1062     return
1063 }
1064
1065 // **************************
1066 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1067 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1068 // builds will do a couple extra steps.
1069 // **************************
1070
1071 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1072 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1073 // it's just used for internal builds
1074 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1075 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1076     if (isWindowsBuildOnlyJob) {
1077         return
1078     }
1079
1080     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1081     // Non pull request builds.
1082     if (!isPR) {
1083         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1084         return
1085     }
1086
1087      def arm64Users = [
1088         'AndyAyersMS',
1089         'briansull',
1090         'BruceForstall',
1091         'CarolEidt',
1092         'cmckinsey',
1093         'erozenfeld',
1094         'janvorli',
1095         'jashook',
1096         'JosephTremoulet',
1097         'pgodeq',
1098         'pgavlin',
1099         'rartemev',
1100         'russellhadley',
1101         'RussKeldorph',
1102         'sandreenko',
1103         'sdmaclea',
1104         'swaroop-sridhar',
1105         'jkotas',
1106         'markwilkie',
1107         'weshaggard'
1108     ]
1109
1110     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1111     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1112     def osGroup = getOSGroup(os)
1113     switch (architecture) {
1114         case 'x64': // editor brace matching: {
1115             if (scenario == 'formatting') {
1116                 assert configuration == 'Checked'
1117                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1118                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1119                 }
1120                 break
1121             }
1122
1123             switch (os) {
1124                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1125                 case 'Debian8.4':
1126                 case 'RHEL7.2':
1127                     if (scenario == 'default') {
1128                         assert !isFlowJob
1129                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1130                     }
1131                     break
1132                 case 'Ubuntu16.04':
1133                     assert !isFlowJob
1134                     assert scenario == 'default'
1135                     // Distinguish with the other architectures (arm and x86)
1136                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1137                     break
1138                 case 'Fedora24':
1139                 case 'Ubuntu16.10':
1140                     assert !isFlowJob
1141                     assert scenario == 'default'
1142                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1143                     break
1144                 case 'Ubuntu':
1145                     if (scenario == 'illink') {
1146                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1147                         break
1148                     }
1149                     // fall through
1150                 case 'OSX10.12':
1151                     // Triggers on the non-flow jobs aren't necessary here
1152                     // Corefx testing uses non-flow jobs.
1153                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1154                         break
1155                     }
1156                     switch (scenario) {
1157                         case 'default':
1158                             // OSX uses checked for default PR tests
1159                             if (configuration == 'Checked') {
1160                                 // Default trigger
1161                                 assert !job.name.contains("centos")
1162                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1163                             }
1164                             break
1165                         case 'jitdiff':
1166                             if (configuration == 'Checked') {
1167                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1168                             }
1169                             break
1170                         case 'ilrt':
1171                             if (configuration == 'Release') {
1172                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1173                             }
1174                             break
1175                         case 'longgc':
1176                             if (configuration == 'Release') {
1177                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1178                             }
1179                             break
1180                         case 'gcsimulator':
1181                             if (configuration == 'Release') {
1182                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1183                             }
1184                             break
1185                         case 'standalone_gc':
1186                             if (configuration == 'Release' || configuration == 'Checked') {
1187                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1188                             }
1189                             break
1190                         case 'gc_reliability_framework':
1191                             if (configuration == 'Release' || configuration == 'Checked') {
1192                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1193                             }
1194                             break
1195                         default:
1196                             if (isJitStressScenario(scenario)) {
1197                                 def displayStr = getStressModeDisplayName(scenario)
1198                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1199                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1200                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1201                             }
1202                             else if (isR2RScenario(scenario)) {
1203                                 if (configuration == 'Release' || configuration == 'Checked') {
1204                                     def displayStr = getR2RDisplayName(scenario)
1205                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1206                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1207                                 }
1208                             }
1209                             else {
1210                                 println("Unknown scenario: ${scenario}");
1211                                 assert false
1212                             }
1213                             break
1214                     }
1215                     break
1216
1217                 case 'CentOS7.1':
1218                     switch (scenario) {
1219                         case 'default':
1220                             // CentOS uses checked for default PR tests while debug is build only
1221                             if (configuration == 'Debug') {
1222                                 // Default trigger
1223                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1224                             }
1225
1226                             // Make sure this is a flow job to get build and test.
1227                             if (configuration == 'Checked' && isFlowJob) {
1228                                 assert job.name.contains("flow")
1229                                 // Default trigger
1230                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1231                             }
1232                             break
1233                         default:
1234                             if (isR2RScenario(scenario)) {
1235                                 if (configuration == 'Release' || configuration == 'Checked') {
1236                                     def displayStr = getR2RDisplayName(scenario)
1237                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1238                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1239                                 }
1240                             }
1241                             break
1242                     }
1243                     break
1244
1245                 case 'Windows_NT':
1246                     switch (scenario) {
1247                         case 'default':
1248                             // Default trigger
1249                             if (configuration == 'Checked') {
1250                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1251                             }
1252                             break
1253                         case 'jitdiff':
1254                             if (configuration == 'Checked') {
1255                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1256                             }
1257                             break
1258                         case 'ilrt':
1259                             if (configuration == 'Release') {
1260                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1261                             }
1262                             break
1263                         case 'longgc':
1264                             if (configuration == 'Release') {
1265                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1266                             }
1267                             break
1268                         case 'gcsimulator':
1269                             if (configuration == 'Release') {
1270                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1271                             }
1272                             break
1273                         case 'standalone_gc':
1274                             if (configuration == 'Release' || configuration == 'Checked') {
1275                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1276                             }
1277                             break
1278                         case 'gc_reliability_framework':
1279                             if (configuration == 'Release' || configuration == 'Checked') {
1280                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1281                             }
1282                             break
1283                         case 'illink':
1284                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1285                             break
1286                         default:
1287                             if (isJitStressScenario(scenario)) {
1288                                 def displayStr = getStressModeDisplayName(scenario)
1289                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1290                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1291                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1292                             }
1293                             else if (isR2RScenario(scenario)) {
1294                                 if (configuration == 'Release' || configuration == 'Checked') {
1295                                     def displayStr = getR2RDisplayName(scenario)
1296                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1297                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1298                                 }
1299                             }
1300                             else {
1301                                 println("Unknown scenario: ${scenario}");
1302                                 assert false
1303                             }
1304                             break
1305                     }
1306                     break
1307                 default:
1308                     println("Unknown os: ${os}");
1309                     assert false
1310                     break
1311             }
1312             break
1313         // editor brace matching: }
1314         case 'armlb':
1315         case 'arm': // editor brace matching: {
1316             switch (os) {
1317                 case 'Ubuntu':
1318                 case 'Ubuntu16.04':
1319                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1320                         break
1321                     }
1322                     assert scenario == 'default'
1323                     job.with {
1324                         publishers {
1325                             azureVMAgentPostBuildAction {
1326                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1327                             }
1328                         }
1329                     }
1330                     if (configuration == 'Debug') {
1331                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1332                     }
1333                     else {
1334                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1335                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1336                     }
1337                     break
1338                 case 'Tizen':
1339                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1340                         break
1341                     }
1342                     architecture='armel'
1343                     job.with {
1344                         publishers {
1345                             azureVMAgentPostBuildAction {
1346                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1347                             }
1348                         }
1349                     }
1350                     if (configuration == 'Checked') {
1351                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1352                     }
1353                     else {
1354                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1355                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1356                     }
1357                     break
1358                 case 'Windows_NT':
1359                     // Triggers on the non-flow jobs aren't necessary here
1360                     if (!isFlowJob) {
1361                         break
1362                     }
1363
1364                     // Set up a private trigger
1365                     def contextString = "${os} ${architecture} Cross ${configuration}"
1366                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1367                     if (scenario == 'default') {
1368                         contextString += " Innerloop"
1369                         triggerString += "\\W+Innerloop"
1370                     }
1371                     else {
1372                         contextString += " ${scenario}"
1373                         triggerString += "\\W+${scenario}"
1374                     }
1375
1376                     if (configuration == 'Debug') {
1377                         contextString += " Build"
1378                         triggerString += "\\W+Build"
1379                     } else {
1380                         contextString += " Build and Test"
1381                         triggerString += "\\W+Build and Test"
1382                     }
1383
1384                     triggerString += ".*"
1385
1386                     switch (scenario) {
1387                         case 'default':
1388                             // Only Checked is a default trigger.
1389                             if (configuration == 'Checked')
1390                             {
1391                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1392                             }
1393                             else
1394                             {
1395                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1396                             }
1397                             break
1398                         default:
1399                             // Stress jobs will use this code path.
1400                             if (isArmWindowsScenario(scenario)) {
1401                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1402                             }
1403                             break
1404                     }
1405                     break
1406                 default:
1407                     println("NYI os: ${os}");
1408                     assert false
1409                     break
1410             }
1411             break
1412         // editor brace matching: }
1413         case 'arm64': // editor brace matching: {
1414             // Set up a private trigger
1415             def contextString = "${os} ${architecture} Cross ${configuration}"
1416             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1417             if (scenario == 'default') {
1418                 contextString += " Innerloop"
1419                 triggerString += "\\W+Innerloop"
1420             }
1421             else {
1422                 contextString += " ${scenario}"
1423                 triggerString += "\\W+${scenario}"
1424             }
1425
1426             if (configuration == 'Debug') {
1427                 contextString += " Build"
1428                 triggerString += "\\W+Build"
1429             } else {
1430                 contextString += " Build and Test"
1431                 triggerString += "\\W+Build and Test"
1432             }
1433
1434             triggerString += ".*"
1435
1436             switch (os) {
1437                 case 'Ubuntu':
1438                 case 'Ubuntu16.04':
1439                     switch (scenario) {
1440                         case 'default':
1441                             if (configuration == 'Debug' && !isFlowJob) {
1442                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1443                             }
1444                             else {
1445                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1446                             }
1447
1448                             break
1449                         default:
1450                             if (isR2RScenario(scenario)) {
1451                                 if (configuration == 'Checked' || configuration == 'Release') {
1452                                     def displayStr = getR2RDisplayName(scenario)
1453                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1454                                 }
1455                             }
1456                             break
1457                     }
1458                     break
1459                 case 'Windows_NT':
1460                     // Triggers on the non-flow jobs aren't necessary here
1461                     if (!isFlowJob) {
1462                         break
1463                     }
1464
1465                     assert isArmWindowsScenario(scenario)
1466                     switch (scenario) {
1467                         case 'default':
1468                             if (configuration == 'Checked') {
1469                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1470                             }
1471                             else {
1472                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1473                             }
1474                             break
1475                         default:
1476                             // Stress jobs will use this code path.
1477                             if (isArmWindowsScenario(scenario)) {
1478                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1479                             }
1480                             break
1481                     }
1482                     break
1483                 default:
1484                     println("NYI os: ${os}");
1485                     assert false
1486                     break
1487             }
1488             break
1489         // editor brace matching: }
1490         case 'x86': // editor brace matching: {
1491             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1492             if (os == 'Ubuntu') {
1493                 // Triggers on the non-flow jobs aren't necessary here
1494                 if (!isFlowJob) {
1495                     break
1496                 }
1497                 // on-demand only for ubuntu x86
1498                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1499                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1500                 break
1501             }
1502             switch (scenario) {
1503                 case 'default':
1504                     if (configuration == 'Checked') {
1505                         assert !job.name.contains("centos")
1506                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1507                     }
1508                     else {
1509                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1510                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1511                     }
1512                     break
1513                 case 'ilrt':
1514                     if (configuration == 'Release') {
1515                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1516                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1517                     }
1518                     break
1519                 case 'longgc':
1520                     if (configuration == 'Release') {
1521                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1522                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1523                     }
1524                     break
1525                 case 'gcsimulator':
1526                     if (configuration == 'Release') {
1527                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1528                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1529                     }
1530                     break
1531                 case 'standalone_gc':
1532                     if (configuration == 'Release' || configuration == 'Checked') {
1533                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1534                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1535                     }
1536                     break
1537                 case 'illink':
1538                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1539                     break
1540                 default:
1541                     if (isJitStressScenario(scenario)) {
1542                         def displayStr = getStressModeDisplayName(scenario)
1543                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1544                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1545                     }
1546                     else if (isR2RScenario(scenario)) {
1547                         if (configuration == 'Release' || configuration == 'Checked') {
1548                             def displayStr = getR2RDisplayName(scenario)
1549                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1550                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1551                         }
1552                     }
1553                     else {
1554                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1555                         assert false
1556                     }
1557                     break
1558             }
1559             break
1560          // editor brace matching: }
1561         case 'x64_arm64_altjit':
1562         case 'x86_arm_altjit': // editor brace matching: {
1563             assert (os == 'Windows_NT')
1564             switch (scenario) {
1565                 case 'default':
1566                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1567                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1568                     break
1569                 default:
1570                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1571                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1572                     break
1573             }
1574             break
1575         // editor brace matching: }
1576         default:
1577             println("Unknown architecture: ${architecture}");
1578             assert false
1579             break
1580     }
1581 }
1582
1583 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1584     def buildCommands = [];
1585     def osGroup = getOSGroup(os)
1586     def lowerConfiguration = configuration.toLowerCase()
1587
1588     def priority = '1'
1589     if (scenario == 'default' && isPR == true) {
1590         priority = '0'
1591     }
1592
1593     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1594
1595     def enableCorefxTesting = isCoreFxScenario(scenario)
1596
1597     // Calculate the build steps, archival, and xunit results
1598     switch (os) {
1599         case 'Windows_NT': // editor brace matching: {
1600             switch (architecture) {
1601                 case 'x64':
1602                 case 'x86':
1603                 case 'x86_arm_altjit':
1604                 case 'x64_arm64_altjit':
1605                     def arch = architecture
1606                     def buildOpts = ''
1607                     if (architecture == 'x86_arm_altjit') {
1608                         arch = 'x86'
1609                     }
1610                     else if (architecture == 'x64_arm64_altjit') {
1611                         arch = 'x64'
1612                     }
1613
1614                     if (scenario == 'formatting') {
1615                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1616                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1617                         break
1618                     }
1619
1620                     if (scenario == 'illink') {
1621                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1622                     }
1623
1624                     // If it is a release build for windows, ensure PGO is used, else fail the build
1625                     if ((lowerConfiguration == 'release') &&
1626                         (scenario in Constants.basicScenarios) &&
1627                         (architecture != 'x86_arm_altjit') &&
1628                         (architecture != 'x64_arm64_altjit')) {
1629
1630                         buildOpts += ' -enforcepgo'
1631                     }
1632
1633                     if (enableCorefxTesting) {
1634                         buildOpts += ' skiptests';
1635                     } else {
1636                         buildOpts += " -priority=${priority}"
1637                     }
1638
1639                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1640                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1641                     // 35 characters long.
1642
1643                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1644
1645                     if (!isBuildOnly) {
1646                         def runtestArguments = ''
1647                         def testOpts = 'collectdumps'
1648
1649                         if (isR2RScenario(scenario)) {
1650
1651                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1652                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1653                             // to cause the tests to be crossgen'ed.
1654
1655                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1656                                 testOpts += ' crossgenaltjit protononjit.dll'
1657                             } else {
1658                                 testOpts += ' crossgen'
1659                             }
1660
1661                             testOpts += ' runcrossgentests'
1662                         }
1663                         else if (scenario == 'jitdiff') {
1664                             testOpts += ' jitdisasm crossgen'
1665                         }
1666                         else if (scenario == 'ilrt') {
1667                             testOpts += ' ilasmroundtrip'
1668                         }
1669                         else if (isLongGc(scenario)) {
1670                             testOpts += " ${scenario} sequential"
1671                         }
1672                         else if (scenario == 'standalone_gc') {
1673                             testOpts += ' gcname clrgc.dll'
1674                         }
1675                         else if (scenario == 'illink') {
1676                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1677                         }
1678
1679                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1680                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1681                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1682                         // which is read by the xunit harness.)
1683                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1684                         {
1685                             def timeout = 1800000
1686                             testOpts += " timeout ${timeout}"
1687                         }
1688
1689                         // If we are running a stress mode, we should write out the set of key
1690                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1691
1692                         def envScriptPath = ''
1693                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1694                             def buildCommandsStr = ''
1695                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1696                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1697
1698                             if (isJitStressScenario(scenario)) {
1699                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1700                             }
1701                             else if (isR2RStressScenario(scenario)) {
1702                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1703                             }
1704
1705                             if (architecture == 'x86_arm_altjit') {
1706                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1707                             }
1708                             else if (architecture == 'x64_arm64_altjit') {
1709                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1710                             }
1711
1712                             envScriptFinalize(os, envScriptPath)
1713
1714                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1715                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1716                             // into a single script.
1717                             buildCommands += buildCommandsStr
1718                         }
1719                         else if (architecture == 'x86_arm_altjit') {
1720                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1721                         }
1722                         else if (architecture == 'x64_arm64_altjit') {
1723                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1724                         }
1725                         if (envScriptPath != '') {
1726                             testOpts += " TestEnv ${envScriptPath}"
1727                         }
1728
1729                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1730
1731                         if (enableCorefxTesting) {
1732                             def workspaceRelativeFxRoot = "_/fx"
1733                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1734
1735                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1736
1737                             // Archive and process (only) the test results
1738                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1739                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1740
1741                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1742                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1743                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1744                         }
1745                         else if (isGcReliabilityFramework(scenario)) {
1746                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1747                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1748                         }
1749                         else {
1750                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1751                         }
1752                     }
1753
1754                     if (!enableCorefxTesting) {
1755                         // Run the rest of the build
1756                         // Build the mscorlib for the other OS's
1757                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1758                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1759
1760                         if (arch == "x64") {
1761                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1762                         }
1763
1764                         // Zip up the tests directory so that we don't use so much space/time copying
1765                         // 10s of thousands of files around.
1766                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1767
1768                         if (!isJitStressScenario(scenario)) {
1769                             // For windows, pull full test results and test drops for x86/x64.
1770                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1771                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1772                         }
1773
1774                         if (scenario == 'jitdiff') {
1775                             // retrive jit-dasm output for base commit, and run jit-diff
1776                             if (!isBuildOnly) {
1777                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1778                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1779                             }
1780                         }
1781
1782                         if (!isBuildOnly) {
1783                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1784                         }
1785                     }
1786                     break
1787                 case 'armlb':
1788                 case 'arm':
1789                     assert isArmWindowsScenario(scenario)
1790
1791                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1792                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1793
1794                     def buildArchitecture = 'arm'
1795
1796                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1797                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1798                     def armCrossgenOpt = ''
1799                     if (architecture == 'armlb') {
1800                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1801                     }
1802
1803                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1804                     priority = '1'
1805
1806                     // This is now a build only job. Do not run tests. Use the flow job.
1807                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1808
1809                     // Zip up the tests directory so that we don't use so much space/time copying
1810                     // 10s of thousands of files around.
1811                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1812
1813                     // Add archival.
1814                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1815                     break
1816                 case 'arm64':
1817                     assert isArmWindowsScenario(scenario)
1818
1819                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1820                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1821
1822                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1823                     priority = '1'
1824
1825                     // This is now a build only job. Do not run tests. Use the flow job.
1826                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1827
1828                     // Zip up the tests directory so that we don't use so much space/time copying
1829                     // 10s of thousands of files around.
1830                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1831
1832                     // Add archival.
1833                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1834                     break
1835                 default:
1836                     println("Unknown architecture: ${architecture}");
1837                     assert false
1838                     break
1839             }
1840             break
1841         // editor brace matching: }
1842         case 'Ubuntu':
1843         case 'Ubuntu16.04':
1844         case 'Ubuntu16.10':
1845         case 'Debian8.4':
1846         case 'OSX10.12':
1847         case 'CentOS7.1':
1848         case 'RHEL7.2':
1849         case 'Tizen':
1850         case 'Fedora24': // editor brace matching: {
1851             switch (architecture) {
1852                 case 'x64':
1853                 case 'x86':
1854                     if (architecture == 'x86' && os == 'Ubuntu') {
1855                         // build and PAL test
1856                         def dockerImage = getDockerImageName(architecture, os, true)
1857                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1858                         dockerImage = getDockerImageName(architecture, os, false)
1859                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1860                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1861                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1862                         break
1863                     }
1864
1865                     if (scenario == 'formatting') {
1866                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1867                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1868                         break
1869                     }
1870
1871                     if (scenario == 'illink') {
1872                         assert(os == 'Ubuntu')
1873                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1874                     }
1875
1876                     if (!enableCorefxTesting) {
1877                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1878                         // only on supported OS platforms.
1879                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1880                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1881
1882                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1883                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1884
1885                         // Basic archiving of the build
1886                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1887                         // And pal tests
1888                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1889                     }
1890                     else {
1891                         // Corefx stress testing
1892                         assert os == 'Ubuntu'
1893                         assert architecture == 'x64'
1894                         assert lowerConfiguration == 'checked'
1895                         assert isJitStressScenario(scenario)
1896
1897                         // Build coreclr
1898                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1899
1900                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1901
1902                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1903                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1904                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1905                         buildCommands += envScriptCmds
1906
1907                         // Build and text corefx
1908                         def workspaceRelativeFxRoot = "_/fx"
1909                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1910
1911                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1912
1913                         // Archive and process (only) the test results
1914                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1915                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1916                     }
1917                     break
1918                 case 'arm64':
1919                     if (!enableCorefxTesting) {
1920                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1921
1922                         // HACK -- Arm64 does not have corefx jobs yet.
1923                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1924                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1925                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1926                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1927
1928                         // Basic archiving of the build
1929                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1930                     }
1931                     break
1932                 case 'arm':
1933                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1934                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1935
1936                     // default values for Ubuntu
1937                     def arm_abi="arm"
1938                     def linuxCodeName="trusty"
1939                     if (os == 'Ubuntu16.04') {
1940                         linuxCodeName="xenial"
1941                     }
1942                     else if (os == 'Tizen') {
1943                         arm_abi="armel"
1944                         linuxCodeName="tizen"
1945                     }
1946
1947                     // Unzip the Windows test binaries first. Exit with 0
1948                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1949
1950                     // Unpack the corefx binaries
1951                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1952                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1953                     if (os != 'Tizen') {
1954                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1955                     }
1956                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1957                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1958                     if (os == 'Ubuntu') {
1959                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1960                     }
1961
1962                     // Call the ARM CI script to cross build and test using docker
1963                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1964                     --mode=docker \\
1965                     --${arm_abi} \\
1966                     --linuxCodeName=${linuxCodeName} \\
1967                     --buildConfig=${lowerConfiguration} \\
1968                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1969                     --coreFxBinDir=./bin/CoreFxBinDir \\
1970                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1971
1972                     // Basic archiving of the build, no pal tests
1973                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1974                     break
1975                 default:
1976                     println("Unknown architecture: ${architecture}");
1977                     assert false
1978                     break
1979             }
1980             break
1981         // editor brace matching: }
1982         default:
1983             println("Unknown os: ${os}");
1984             assert false
1985             break
1986     } // os
1987
1988     return buildCommands
1989 }
1990
1991 Constants.allScenarios.each { scenario ->
1992     [true, false].each { isPR ->
1993         Constants.architectureList.each { architecture ->
1994             Constants.configurationList.each { configuration ->
1995                 Constants.osList.each { os ->
1996                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1997                     // and reset the os to Windows_NT
1998                     def isBuildOnly = false
1999                     if (os == 'Windows_NT_BuildOnly') {
2000                         isBuildOnly = true
2001                         os = 'Windows_NT'
2002                     }
2003
2004                     // Tizen is only supported for arm architecture
2005                     if (os == 'Tizen' && architecture != 'arm') {
2006                         return
2007                     }
2008
2009                     // Skip totally unimplemented (in CI) configurations.
2010                     switch (architecture) {
2011                         case 'arm64':
2012                             if (os == 'Ubuntu16.04') {
2013                                 os = 'Ubuntu'
2014                             }
2015
2016                             // Windows and Ubuntu only
2017                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2018                                 return
2019                             }
2020                             break
2021                         case 'arm':
2022                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2023                                 return
2024                             }
2025                             break
2026                         case 'armlb':
2027                             if (os != 'Windows_NT') {
2028                                 return
2029                             }
2030                             break
2031                         case 'x86':
2032                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2033                                 return
2034                             }
2035                             break
2036                         case 'x86_arm_altjit':
2037                         case 'x64_arm64_altjit':
2038                             if (os != 'Windows_NT') {
2039                                 return
2040                             }
2041                             break
2042                         case 'x64':
2043                             // Everything implemented
2044                             break
2045                         default:
2046                             println("Unknown architecture: ${architecture}")
2047                             assert false
2048                             break
2049                     }
2050
2051                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2052                     // with checked builds)
2053                     if (isJitStressScenario(scenario)) {
2054                         if (configuration != 'Checked') {
2055                             return
2056                         }
2057
2058                         // Since these are just execution time differences,
2059                         // skip platforms that don't execute the tests here (Windows_NT only)
2060                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2061                         if (!isEnabledOS || isBuildOnly) {
2062                             return
2063                         }
2064
2065                         switch (architecture) {
2066                             case 'x64':
2067                             case 'x86':
2068                             case 'x86_arm_altjit':
2069                             case 'x64_arm64_altjit':
2070                                 // x86 ubuntu: default only
2071                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2072                                     return
2073                                 }
2074                                 // Windows: Everything implemented
2075                                 break
2076
2077                             default:
2078                                 // arm, arm64, armlb: stress is handled through flow jobs.
2079                                 return
2080                         }
2081                     }
2082                     else if (isR2RScenario(scenario)) {
2083                         if (os != 'Windows_NT') {
2084                             return
2085                         }
2086                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2087                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2088                             return
2089                         }
2090                     }
2091                     else {
2092                         // Skip scenarios
2093                         switch (scenario) {
2094                             case 'ilrt':
2095                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2096                                 // the default scenario build
2097                                 if (os != 'Windows_NT') {
2098                                     return
2099                                 }
2100                                 // Only x64 for now
2101                                 if (architecture != 'x64') {
2102                                     return
2103                                 }
2104                                 // Release only
2105                                 if (configuration != 'Release') {
2106                                     return
2107                                 }
2108                                 break
2109                             case 'jitdiff':
2110                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2111                                     return
2112                                 }
2113                                 if (architecture != 'x64') {
2114                                     return
2115                                 }
2116                                 if (configuration != 'Checked') {
2117                                     return
2118                                 }
2119                                 break
2120                             case 'longgc':
2121                             case 'gcsimulator':
2122                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2123                                     return
2124                                 }
2125                                 if (architecture != 'x64') {
2126                                     return
2127                                 }
2128                                 if (configuration != 'Release') {
2129                                     return
2130                                 }
2131                                 break
2132                             case 'gc_reliability_framework':
2133                             case 'standalone_gc':
2134                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2135                                     return
2136                                 }
2137
2138                                 if (architecture != 'x64') {
2139                                     return
2140                                 }
2141
2142                                 if (configuration != 'Release' && configuration != 'Checked') {
2143                                     return
2144                                 }
2145                                 break
2146                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2147                             case 'formatting':
2148                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2149                                     return
2150                                 }
2151                                 if (architecture != 'x64') {
2152                                     return
2153                                 }
2154                                 if (configuration != 'Checked') {
2155                                     return
2156                                 }
2157                                 if (isBuildOnly) {
2158                                     return
2159                                 }
2160                                 break
2161                             case 'illink':
2162                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2163                                     return
2164                                 }
2165                                 if (architecture != 'x64' && architecture != 'x86') {
2166                                     return
2167                                 }
2168                                 if (isBuildOnly) {
2169                                     return
2170                                 }
2171                                 break
2172                             case 'default':
2173                                 // Nothing skipped
2174                                 break
2175                             default:
2176                                 println("Unknown scenario: ${scenario}")
2177                                 assert false
2178                                 break
2179                         }
2180                     }
2181
2182                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2183                     // runtime behavior, not compile-time behavior, are not interesting.
2184                     switch (architecture) {
2185                         case 'x86_arm_altjit':
2186                         case 'x64_arm64_altjit':
2187                             if (isGCStressRelatedTesting(scenario)) {
2188                                 return
2189                             }
2190                             break
2191                         default:
2192                             break
2193                     }
2194
2195                     // Calculate names
2196                     def lowerConfiguration = configuration.toLowerCase()
2197                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2198                     def folderName = getJobFolder(scenario)
2199
2200                     // Create the new job
2201                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2202                     addToViews(newJob, isPR, architecture, os)
2203
2204                     def machineAffinityOptions = null
2205
2206                     if (os != 'Windows_NT') {
2207                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2208                     }
2209                     else {
2210                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2211                     }
2212
2213                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2214
2215                     // Add all the standard options
2216                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2217                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2218
2219                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2220                     def osGroup = getOSGroup(os)
2221
2222                     newJob.with {
2223                         steps {
2224                             if (os == 'Windows_NT') {
2225                                 buildCommands.each { buildCommand ->
2226                                     batchFile(buildCommand)
2227                                 }
2228                             }
2229                             else {
2230                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2231                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2232                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2233                                     // Define the Windows Tests and Corefx build job names
2234                                     def WindowsTestsName = projectFolder + '/' +
2235                                                            Utilities.getFullJobName(project,
2236                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2237                                                                                     false)
2238                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2239                                                        Utilities.getFolderName(branch)
2240
2241                                     // Copy the Windows test binaries and the Corefx build binaries
2242                                     copyArtifacts(WindowsTestsName) {
2243                                         includePatterns('bin/tests/tests.zip')
2244                                         buildSelector {
2245                                             latestSuccessful(true)
2246                                         }
2247                                     }
2248
2249                                     def arm_abi = 'arm'
2250                                     def corefx_os = 'linux'
2251                                     if (os == 'Tizen') {
2252                                         arm_abi = 'armel'
2253                                         corefx_os = 'tizen'
2254                                     }
2255
2256                                     // Let's use release CoreFX to test checked CoreCLR,
2257                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2258                                     def corefx_lowerConfiguration = lowerConfiguration
2259                                     if ( lowerConfiguration == 'checked' ) {
2260                                         corefx_lowerConfiguration='release'
2261                                     }
2262
2263                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2264                                         includePatterns('bin/build.tar.gz')
2265                                         buildSelector {
2266                                             latestSuccessful(true)
2267                                         }
2268                                     }
2269                                 }
2270
2271                                 buildCommands.each { buildCommand ->
2272                                     shell(buildCommand)
2273                                 }
2274                             }
2275                         }
2276                     } // newJob.with
2277
2278                 } // os
2279             } // configuration
2280         } // architecture
2281     } // isPR
2282 } // scenario
2283
2284
2285 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2286 Constants.allScenarios.each { scenario ->
2287     [true, false].each { isPR ->
2288         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2289             Constants.crossList.each { os ->
2290                 if (architecture == 'arm64') {
2291                     if (os != "Ubuntu" && os != "Windows_NT") {
2292                         return
2293                     }
2294                 } else if (architecture == 'arm' || architecture == 'armlb') {
2295                     if (os != 'Windows_NT') {
2296                         return
2297                     }
2298                 }
2299                 else if (architecture == 'x86') {
2300                     if (os != "Ubuntu") {
2301                         return
2302                     }
2303                 }
2304
2305                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2306
2307                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2308                     return
2309                 }
2310
2311                 Constants.configurationList.each { configuration ->
2312
2313                     // First, filter based on OS.
2314
2315                     if (os == 'Windows_NT') {
2316                         if (!isArmWindowsScenario(scenario)) {
2317                             return
2318                         }
2319                     }
2320                     else {
2321                         // Non-Windows
2322                         if (architecture == 'arm64') {
2323                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2324                                 return
2325                             }
2326                         }
2327                         else if (architecture == 'x86') {
2328                             // Linux/x86 only want default test
2329                             if (scenario != 'default') {
2330                                 return
2331                             }
2332                         }
2333                     }
2334
2335                     // For CentOS, we only want Checked/Release builds.
2336                     if (os == 'CentOS7.1') {
2337                         if (configuration != 'Checked' && configuration != 'Release') {
2338                             return
2339                         }
2340                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2341                             return
2342                         }
2343                     }
2344
2345                     // For RedHat and Debian, we only do Release builds.
2346                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2347                         if (configuration != 'Release') {
2348                             return
2349                         }
2350                         if (scenario != 'default') {
2351                             return
2352                         }
2353                     }
2354
2355                     // Next, filter based on scenario.
2356
2357                     if (isJitStressScenario(scenario)) {
2358                         if (configuration != 'Checked') {
2359                             return
2360                         }
2361                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2362                         if (isCoreFxScenario(scenario)) {
2363                             return
2364                         }
2365                     }
2366                     else if (isR2RBaselineScenario(scenario)) {
2367                         if (configuration != 'Checked' && configuration != 'Release') {
2368                             return
2369                         }
2370                     }
2371                     else if (isR2RStressScenario(scenario)) {
2372                         if (configuration != 'Checked') {
2373                             return
2374                         }
2375                     }
2376                     else {
2377                         // Skip scenarios
2378                         switch (scenario) {
2379                             case 'ilrt':
2380                             case 'longgc':
2381                             case 'gcsimulator':
2382                                 // Long GC tests take a long time on non-Release builds
2383                                 // ilrt is also Release only
2384                                 if (configuration != 'Release') {
2385                                     return
2386                                 }
2387                                 break
2388                             case 'jitdiff':
2389                                 if (configuration != 'Checked') {
2390                                     return;
2391                                 }
2392                                 break
2393                             case 'gc_reliability_framework':
2394                             case 'standalone_gc':
2395                                 if (configuration != 'Release' && configuration != 'Checked') {
2396                                     return
2397                                 }
2398                                 break
2399                             case 'formatting':
2400                                 return
2401                             case 'illink':
2402                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2403                                     return
2404                                 }
2405                                 break
2406                             case 'default':
2407                                 // Nothing skipped
2408                                 break
2409                             default:
2410                                 println("Unknown scenario: ${scenario}")
2411                                 assert false
2412                                 break
2413                         }
2414                     }
2415
2416                     // Done filtering. Now, create the jobs.
2417
2418                     def lowerConfiguration = configuration.toLowerCase()
2419                     def osGroup = getOSGroup(os)
2420                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2421
2422                     def inputCoreCLRBuildName = projectFolder + '/' +
2423                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2424
2425                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2426                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2427                     def testBuildScenario = scenario
2428                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2429                         testBuildScenario = 'default'
2430                     }
2431
2432                     def inputWindowsTestBuildArch = architecture
2433                     if (architecture == "arm64" && os != "Windows_NT") {
2434                         // Use the x64 test build for arm64 unix
2435                         inputWindowsTestBuildArch = "x64"
2436                     }
2437
2438                     def inputWindowsTestsBuildName = ""
2439
2440                     if (isJitStressScenario(scenario)) {
2441                         inputWindowsTestsBuildName = projectFolder + '/' +
2442                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2443                     } else {
2444                         inputWindowsTestsBuildName = projectFolder + '/' +
2445                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2446
2447                     }
2448
2449
2450                     // Enable Server GC for Ubuntu PR builds
2451                     def serverGCString = ''
2452                     if (os == 'Ubuntu' && isPR) {
2453                         serverGCString = '--useServerGC'
2454                     }
2455
2456                     def testOpts = ''
2457
2458                     if (isR2RScenario(scenario)) {
2459
2460                         testOpts += ' --crossgen --runcrossgentests'
2461
2462                         if (scenario == 'r2r_jitstress1') {
2463                             testOpts += ' --jitstress=1'
2464                         }
2465                         else if (scenario == 'r2r_jitstress2') {
2466                             testOpts += ' --jitstress=2'
2467                         }
2468                         else if (scenario == 'r2r_jitstressregs1') {
2469                             testOpts += ' --jitstressregs=1'
2470                         }
2471                         else if (scenario == 'r2r_jitstressregs2') {
2472                             testOpts += ' --jitstressregs=2'
2473                         }
2474                         else if (scenario == 'r2r_jitstressregs3') {
2475                             testOpts += ' --jitstressregs=3'
2476                         }
2477                         else if (scenario == 'r2r_jitstressregs4') {
2478                             testOpts += ' --jitstressregs=4'
2479                         }
2480                         else if (scenario == 'r2r_jitstressregs8') {
2481                             testOpts += ' --jitstressregs=8'
2482                         }
2483                         else if (scenario == 'r2r_jitstressregs0x10') {
2484                             testOpts += ' --jitstressregs=0x10'
2485                         }
2486                         else if (scenario == 'r2r_jitstressregs0x80') {
2487                             testOpts += ' --jitstressregs=0x80'
2488                         }
2489                         else if (scenario == 'r2r_jitstressregs0x1000') {
2490                             testOpts += ' --jitstressregs=0x1000'
2491                         }
2492                         else if (scenario == 'r2r_jitminopts') {
2493                             testOpts += ' --jitminopts'
2494                         }
2495                         else if (scenario == 'r2r_jitforcerelocs') {
2496                             testOpts += ' --jitforcerelocs'
2497                         }
2498                         else if (scenario == 'r2r_gcstress15') {
2499                             testOpts += ' --gcstresslevel=0xF'
2500                         }
2501                     }
2502                     else if (scenario == 'jitdiff') {
2503                         testOpts += ' --jitdisasm --crossgen'
2504                     }
2505                     else if (scenario == 'illink') {
2506                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2507                     }
2508                     else if (isLongGc(scenario)) {
2509                         // Long GC tests behave very poorly when they are not
2510                         // the only test running (many of them allocate until OOM).
2511                         testOpts += ' --sequential'
2512
2513                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2514                         // for running long GC and GCSimulator tests, respectively. We don't use them
2515                         // here because using a playlist file produces much more readable output on the CI machines
2516                         // and reduces running time.
2517                         //
2518                         // The Long GC playlist contains all of the tests that are
2519                         // going to be run. The GCSimulator playlist contains all of
2520                         // the GC simulator tests.
2521                         if (scenario == 'longgc') {
2522                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2523                         }
2524                         else if (scenario == 'gcsimulator') {
2525                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2526                         }
2527                     }
2528                     else if (isGcReliabilityFramework(scenario)) {
2529                         testOpts += ' --build-overlay-only'
2530                     }
2531                     else if (scenario == 'standalone_gc') {
2532                         if (osGroup == 'OSX') {
2533                             testOpts += ' --gcname=libclrgc.dylib'
2534                         }
2535                         else if (osGroup == 'Linux') {
2536                             testOpts += ' --gcname=libclrgc.so'
2537                         }
2538                         else {
2539                             println("Unexpected OS group: ${osGroup} for os ${os}")
2540                             assert false
2541                         }
2542                     }
2543
2544                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2545
2546                     def folder = getJobFolder(scenario)
2547                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2548                         // Add parameters for the inputs
2549
2550                         if (windowsArmJob == true) {
2551                             parameters {
2552                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2553                             }
2554                         }
2555                         else {
2556                             parameters {
2557                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2558                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2559                             }
2560                         }
2561
2562                         steps {
2563                             // Set up the copies
2564
2565                             // Coreclr build containing the tests and mscorlib
2566                             // pri1 jobs still need to copy windows_nt built tests
2567                             if (windowsArmJob != true) {
2568                                 copyArtifacts(inputWindowsTestsBuildName) {
2569                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2570                                     buildSelector {
2571                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2572                                     }
2573                                 }
2574                             }
2575
2576                             // Coreclr build we are trying to test
2577                             //
2578                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2579
2580                             copyArtifacts(inputCoreCLRBuildName) {
2581                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2582                                 buildSelector {
2583                                     buildNumber('${CORECLR_BUILD}')
2584                                 }
2585                             }
2586
2587                             // Windows CoreCLR Arm(64) will restore corefx
2588                             // packages correctly.
2589                             //
2590                             // In addition, test steps are entirely different
2591                             // because we do not have a unified runner
2592                             if (windowsArmJob != true) {
2593                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2594
2595                                 // HACK -- Arm64 does not have corefx jobs yet.
2596                                 // Clone corefx and build the native packages overwriting the x64 packages.
2597                                 if (architecture == 'arm64') {
2598                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2599                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2600                                 }
2601                                 else if (architecture == 'x86') {
2602                                     shell("mkdir ./bin/CoreFxNative")
2603
2604                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2605                                         includePatterns('bin/build.tar.gz')
2606                                         targetDirectory('bin/CoreFxNative')
2607                                         buildSelector {
2608                                             latestSuccessful(true)
2609                                         }
2610                                     }
2611
2612                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2613                                 }
2614
2615                                 // Unzip the tests first.  Exit with 0
2616                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2617                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2618
2619                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2620
2621                                 // Execute the tests
2622                                 def runDocker = isNeedDocker(architecture, os, false)
2623                                 def dockerPrefix = ""
2624                                 def dockerCmd = ""
2625                                 if (runDocker) {
2626                                     def dockerImage = getDockerImageName(architecture, os, false)
2627                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2628                                     dockerCmd = dockerPrefix + "${dockerImage} "
2629                                 }
2630
2631                                 // If we are running a stress mode, we'll set those variables first
2632                                 def testEnvOpt = ""
2633                                 if (isJitStressScenario(scenario)) {
2634                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2635                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2636                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2637                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2638                                     shell("${envScriptCmds}")
2639                                     testEnvOpt = "--test-env=" + scriptFileName
2640                                 }
2641
2642                                 if (isGCStressRelatedTesting(scenario)) {
2643                                     shell('./init-tools.sh')
2644                                 }
2645
2646                                 shell("""${dockerCmd}./tests/runtest.sh \\
2647                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2648                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2649                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2650                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2651
2652                                 if (isGcReliabilityFramework(scenario)) {
2653                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2654                                     if (serverGCString != '') {
2655                                         if (runDocker) {
2656                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2657                                         }
2658                                         else {
2659                                             shell("export COMPlus_gcServer=1")
2660                                         }
2661                                     }
2662
2663                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2664                                 }
2665                             }
2666
2667                             else { // windowsArmJob == true
2668                                 // Unzip tests.
2669                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2670
2671                                 // Build the build commands
2672                                 def buildCommands = ""
2673
2674                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2675                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2676                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2677
2678                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2679                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2680
2681                                 // For all jobs
2682                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2683
2684                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2685                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2686
2687                                 // ARM legacy backend; this is an altjit.
2688                                 if (architecture == "armlb") {
2689                                     addEnvVariable("COMPlus_AltJit", "*")
2690                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2691                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2692                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2693                                 }
2694
2695                                 // If we are running a stress mode, we'll set those variables as well
2696                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2697                                     def stressValues = null
2698                                     if (isJitStressScenario(scenario)) {
2699                                         stressValues = Constants.jitStressModeScenarios[scenario]
2700                                     }
2701                                     else {
2702                                         stressValues = Constants.r2rStressScenarios[scenario]
2703                                     }
2704
2705                                     stressValues.each { key, value ->
2706                                         addEnvVariable(key, value)
2707                                     }
2708                                 }
2709
2710                                 if (isR2RScenario(scenario)) {
2711                                     // Crossgen the framework assemblies.
2712                                     buildCommands += """
2713 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2714 @goto skip_PrecompileAssembly
2715
2716 :PrecompileAssembly
2717 @REM Skip mscorlib since it is already precompiled.
2718 @if /I "%3" == "mscorlib.dll" exit /b 0
2719 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2720
2721 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2722 @if "%errorlevel%" == "-2146230517" (
2723     echo %2 is not a managed assembly.
2724 ) else if "%errorlevel%" == "-2146234344" (
2725     echo %2 is not a managed assembly.
2726 ) else if %errorlevel% neq 0 (
2727     echo Unable to precompile %2
2728 ) else (
2729     echo Precompiled %2
2730 )
2731 @exit /b 0
2732
2733 :skip_PrecompileAssembly
2734 """
2735
2736                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2737                                     // crossgen on tests before running them.
2738                                     addEnvVariable("RunCrossGen", "true")
2739                                 }
2740
2741                                 // Create the smarty command
2742                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2743                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2744                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2745
2746                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2747
2748                                 if (architecture == "armlb") {
2749                                     addExclude("LEGACYJIT_FAIL")
2750                                 }
2751
2752                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2753                                     def failTag = "JITSTRESS_FAIL"
2754                                     def excludeTag = "JITSTRESS_EXCLUDE"
2755
2756                                     if (scenario.contains('gc')) {
2757                                         failTag = "GCSTRESS_FAIL"
2758                                         excludeTag = "GCSTRESS_EXCLUDE"
2759                                     }
2760
2761                                     addArchSpecificExclude(architecture, failTag)
2762                                     addArchSpecificExclude(architecture, excludeTag)
2763                                 }
2764                                 else {
2765                                     addExclude("pri1")
2766                                 }
2767
2768                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2769                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2770                                 addExclude("LONG_RUNNING")
2771
2772                                 smartyCommand += "/lstFile Tests.lst"
2773
2774                                 def testListArch = [
2775                                     'arm64': 'arm64',
2776                                     'arm': 'arm',
2777                                     'armlb': 'arm'
2778                                 ]
2779
2780                                 def archLocation = testListArch[architecture]
2781
2782                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2783                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2784                                 addCommand("${smartyCommand}")
2785
2786                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2787                                 // However, we also need to remove all the variables that were set during this batch file, so we
2788                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2789                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2790                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2791                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2792                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2793                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2794
2795                                 // ZIP up the smarty output, no matter what the smarty result.
2796                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2797
2798                                 addCommand("echo %errorlevel%")
2799                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2800
2801                                 // Use the smarty errorlevel as the script errorlevel.
2802                                 addCommand("exit /b %__save_smarty_errorlevel%")
2803
2804                                 batchFile(buildCommands)
2805                             }
2806                         }
2807                     }
2808
2809                     addToViews(newJob, isPR, architecture, os)
2810
2811                     if (scenario == 'jitdiff') {
2812                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2813                     }
2814
2815                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2816                     if (os in ['Ubuntu']) {
2817                         SummaryBuilder summaries = new SummaryBuilder()
2818                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2819                         summaries.emit(newJob)
2820                     }
2821
2822                     def affinityOptions = null
2823
2824                     if (windowsArmJob == true) {
2825                         affinityOptions = [
2826                             "use_arm64_build_machine" : false
2827                         ]
2828                     }
2829
2830                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2831                         affinityOptions = [
2832                             "large_pages" : false
2833                         ]
2834                     }
2835
2836                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2837                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2838
2839                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2840
2841                     if (windowsArmJob != true) {
2842                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2843                     }
2844                     else {
2845                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2846
2847                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2848                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2849                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2850                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2851                     }
2852
2853                     // Create a build flow to join together the build and tests required to run this test.
2854                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2855                     // Linux CoreCLR test
2856                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2857                     def fullTestJobName = projectFolder + '/' + newJob.name
2858                     // Add a reference to the input jobs for report purposes
2859                     JobReport.Report.addReference(inputCoreCLRBuildName)
2860                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2861                     JobReport.Report.addReference(fullTestJobName)
2862                     def newFlowJob = null
2863
2864                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2865                         // Do not create the flow job for RHEL jobs.
2866                         return
2867                     }
2868
2869                     // For pri0 jobs we can build tests on unix
2870                     if (windowsArmJob) {
2871                         // For Windows arm jobs there is no reason to build a parallel test job.
2872                         // The product build supports building and archiving the tests.
2873
2874                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2875                         buildFlow("""
2876 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2877
2878 // And then build the test build
2879 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2880 """)
2881                         }
2882                     }
2883                     else {
2884                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2885                         buildFlow("""
2886 // Build the input jobs in parallel
2887 parallel (
2888 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2889 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2890 )
2891
2892 // And then build the test build
2893 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2894             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2895 """)
2896                         }
2897                     }
2898
2899                     addToViews(newFlowJob, isPR, architecture, os)
2900
2901                     // For the flow jobs set the machine affinity as x64 if an armarch.
2902                     def flowArch = architecture
2903
2904                     if (flowArch in validWindowsNTCrossArches) {
2905                         flowArch = 'x64'
2906                         affinityOptions = null
2907                     }
2908
2909                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2910                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2911                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2912                 } // configuration
2913             } // os
2914         } // architecture
2915     } // isPR
2916 } // scenario
2917
2918 JobReport.Report.generateJobReport(out)
2919
2920 // Make the call to generate the help job
2921 Utilities.createHelperJob(this, project, branch,
2922     "Welcome to the ${project} Repository",  // This is prepended to the help message
2923     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2924
2925 Utilities.addCROSSCheck(this, project, branch)