netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 // **************************************************************
 258 // Create some specific views
 259 //
 260 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 261 // views based on a single regular expression. These views will be generated by adding a
 262 // specific set of jobs to them.
 263 //
 264 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 265 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 266 // those are required, don't add them (which simplifies the view pages, as well).
 267 // **************************************************************
 268
 269 class Views {
 270     def static MergeJobView = null
 271     def static PeriodicJobView = null
 272     def static ArchitectureViews = [:]
 273     def static OSViews = [:]
 274 }
 275
 276 // MergeJobView: include all jobs that execute when a PR change is merged.
 277 Views.MergeJobView = listView('Merge') {
 278     columns {
 279         status()
 280         weather()
 281         name()
 282         lastSuccess()
 283         lastFailure()
 284         lastDuration()
 285         buildButton()
 286     }
 287 }
 288
 289 // PeriodicJobView: include all jobs that execute on a schedule
 290 Views.PeriodicJobView = listView('Periodic') {
 291     columns {
 292         status()
 293         weather()
 294         name()
 295         lastSuccess()
 296         lastFailure()
 297         lastDuration()
 298         buildButton()
 299     }
 300 }
 301
 302 // Create a view for non-PR jobs for each architecture.
 303 Constants.architectureList.each { architecture ->
 304     Views.ArchitectureViews[architecture] = listView(architecture) {
 305         columns {
 306             status()
 307             weather()
 308             name()
 309             lastSuccess()
 310             lastFailure()
 311             lastDuration()
 312             buildButton()
 313         }
 314     }
 315 }
 316
 317 // Create a view for non-PR jobs for each OS.
 318 Constants.osList.each { os ->
 319     // Don't create one for the special 'Windows_NT_BuildOnly'
 320     if (os == 'Windows_NT_BuildOnly') {
 321         return
 322     }
 323     Views.OSViews[os] = listView(os) {
 324         columns {
 325             status()
 326             weather()
 327             name()
 328             lastSuccess()
 329             lastFailure()
 330             lastDuration()
 331             buildButton()
 332         }
 333     }
 334 }
 335
 336 def static addToMergeView(def job) {
 337     Views.MergeJobView.with {
 338         jobs {
 339             name(job.name)
 340         }
 341     }
 342 }
 343
 344 def static addToPeriodicView(def job) {
 345     Views.PeriodicJobView.with {
 346         jobs {
 347             name(job.name)
 348         }
 349     }
 350 }
 351
 352 def static addToViews(def job, def isPR, def architecture, def os) {
 353     if (isPR) {
 354         // No views want PR jobs currently.
 355         return
 356     }
 357
 358     // Add to architecture view.
 359     Views.ArchitectureViews[architecture].with {
 360         jobs {
 361             name(job.name)
 362         }
 363     }
 364
 365     // Add to OS view.
 366     Views.OSViews[os].with {
 367         jobs {
 368             name(job.name)
 369         }
 370     }
 371 }
 372
 373 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 374     addToPeriodicView(job)
 375     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 376 }
 377
 378 def static addGithubPushTriggerHelper(def job) {
 379     addToMergeView(job)
 380     Utilities.addGithubPushTrigger(job)
 381 }
 382
 383
 384 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 385     assert os instanceof String
 386     assert architecture instanceof String
 387
 388     def armArches = ['arm', 'armlb', 'arm64']
 389     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 390
 391     if (!(architecture in armArches)) {
 392         assert options == null
 393         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 394
 395         return
 396     }
 397
 398     // This is an arm(64) job.
 399     //
 400     // There are several options.
 401     //
 402     // Windows_NT
 403     //
 404     // Arm32 (Build) -> latest-arm64
 405     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 406     // Arm32 (Test)  -> arm64-windows_nt
 407     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 408     //
 409     // Arm64 (Build) -> latest-arm64
 410     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 411     // Arm64 (Test)  -> arm64-windows_nt
 412     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 413     //
 414     // Ubuntu
 415     //
 416     // Arm32 (Build) -> arm-cross-latest
 417     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 418     // Arm32 (Test)  -> NYI Arch not supported
 419     //       |->
 420     //
 421     // Arm64 (Build) -> arm64-cross-latest
 422     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 423     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 424     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 425     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 426     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 427
 428     // This has to be a arm arch
 429     assert architecture in armArches
 430     if (os == "Windows_NT") {
 431         // Arm(64) Windows jobs share the same machines for now
 432         def isBuild = options['use_arm64_build_machine'] == true
 433
 434         if (isBuild == true) {
 435             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 436         } else {
 437             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 438         }
 439     } else {
 440         assert os != 'Windows_NT'
 441         assert os in supportedArmLinuxOs
 442
 443         if (architecture == 'arm' || architecture == 'armlb') {
 444             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 445         } else {
 446             // Arm64 Linux
 447             if (options['is_build_only'] == true) {
 448                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 449             } else {
 450                 // Arm64 Test Machines
 451                 if (options['large_pages'] == false) {
 452                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 453                 } else {
 454                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 455                 }
 456             }
 457         }
 458     }
 459 }
 460
 461 def static isGCStressRelatedTesting(def scenario) {
 462     // The 'r2r_gcstress15' scenario is a basic scenario.
 463     // Detect it and make it a GCStress related.
 464     if (scenario == 'r2r_gcstress15')
 465     {
 466         return true;
 467     }
 468
 469     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 470     def scenarioName = scenario.toLowerCase()
 471     def isGCStressTesting = false
 472     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 473         if (k in gcStressTestEnvVars) {
 474             isGCStressTesting = true;
 475         }
 476     }
 477     return isGCStressTesting
 478 }
 479
 480 def static isCoreFxScenario(def scenario) {
 481     def corefx_prefix = 'corefx_'
 482     if (scenario.length() < corefx_prefix.length()) {
 483         return false
 484     }
 485     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 486 }
 487
 488 def static isR2RBaselineScenario(def scenario) {
 489     return (scenario == 'r2r')
 490 }
 491
 492 def static isR2RStressScenario(def scenario) {
 493     return Constants.r2rStressScenarios.containsKey(scenario)
 494 }
 495
 496 def static isR2RScenario(def scenario) {
 497     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 498 }
 499
 500 def static isJitStressScenario(def scenario) {
 501     return Constants.jitStressModeScenarios.containsKey(scenario)
 502 }
 503
 504 def static isLongGc(def scenario) {
 505     return (scenario == 'longgc' || scenario == 'gcsimulator')
 506 }
 507
 508 def static isJitDiff(def scenario) {
 509     return (scenario == 'jitdiff')
 510 }
 511
 512 def static isGcReliabilityFramework(def scenario) {
 513     return (scenario == 'gc_reliability_framework')
 514 }
 515
 516 def static isArmWindowsScenario(def scenario) {
 517     return Constants.validArmWindowsScenarios.containsKey(scenario)
 518 }
 519
 520 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 521     // 2 hours (120 minutes) is the default timeout
 522     def timeout = 120
 523
 524     if (!(scenario == 'default' && isPR == true)) {
 525         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 526         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 527         timeout = 240
 528     }
 529
 530     if (!isBuildOnly) {
 531         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 532         if (isGCStressRelatedTesting(scenario)) {
 533             timeout = 4320
 534         }
 535         else if (isCoreFxScenario(scenario)) {
 536             timeout = 360
 537         }
 538         else if (isJitStressScenario(scenario)) {
 539             timeout = 240
 540         }
 541         else if (isR2RBaselineScenario(scenario)) {
 542             timeout = 240
 543         }
 544         else if (isLongGc(scenario)) {
 545             timeout = 1440
 546         }
 547         else if (isJitDiff(scenario)) {
 548             timeout = 240
 549         }
 550         else if (isGcReliabilityFramework(scenario)) {
 551             timeout = 1440
 552         }
 553         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 554             timeout = 240
 555         }
 556     }
 557
 558     if (configuration == 'Debug') {
 559         // Debug runs can be very slow. Add an hour.
 560         timeout += 60
 561     }
 562
 563     // If we've changed the timeout from the default, set it in the job.
 564
 565     if (timeout != 120) {
 566         Utilities.setJobTimeout(newJob, timeout)
 567     }
 568 }
 569
 570 def static getJobFolder(def scenario) {
 571     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 572         return 'jitstress'
 573     }
 574     if (scenario == 'illink') {
 575         return 'illink'
 576     }
 577     return ''
 578 }
 579
 580 def static getStressModeDisplayName(def scenario) {
 581     def displayStr = ''
 582     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 583         def prefixLength = 'COMPlus_'.length()
 584         if (k.length() >= prefixLength) {
 585             def modeName = k.substring(prefixLength, k.length())
 586             displayStr += ' ' + modeName + '=' + v
 587         }
 588     }
 589
 590     if (isCoreFxScenario(scenario)) {
 591         displayStr = ('CoreFx ' + displayStr).trim()
 592     }
 593
 594     return displayStr
 595 }
 596
 597 def static getR2RDisplayName(def scenario) {
 598     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 599     def displayStr = scenario
 600     def prefixLength = 'r2r_'.length()
 601     if (displayStr.length() >= prefixLength) {
 602         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 603     } else if (scenario == 'r2r') {
 604         displayStr = "R2R"
 605     }
 606     return displayStr
 607 }
 608
 609 //
 610 // Functions to create an environment script.
 611 //      envScriptCreate -- initialize the script (call first)
 612 //      envScriptFinalize -- finalize the script (call last)
 613 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 614 //      envScriptAppendExistingScript -- append an existing script to the generated script
 615 //
 616 // Each script returns a string of commands. Concatenate all the strings together before
 617 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 618 //
 619
 620 // Initialize the environment setting script.
 621 def static envScriptCreate(def os, def stepScriptLocation) {
 622     def stepScript = ''
 623     if (os == 'Windows_NT') {
 624         stepScript += "echo Creating TestEnv script\r\n"
 625         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 626
 627         // Create at least an empty script.
 628         stepScript += "echo. > ${stepScriptLocation}\r\n"
 629     }
 630     else {
 631         stepScript += "echo Creating environment setting script\n"
 632         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 633     }
 634
 635     return stepScript
 636 }
 637
 638 // Generates the string for setting stress mode variables.
 639 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 640     def stepScript = ''
 641     if (os == 'Windows_NT') {
 642         stressModeVars.each{ k, v ->
 643             // Write out what we are writing to the script file
 644             stepScript += "echo Setting ${k}=${v}\r\n"
 645             // Write out the set itself to the script file`
 646             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 647         }
 648     }
 649     else {
 650         stressModeVars.each{ k, v ->
 651             // Write out what we are writing to the script file
 652             stepScript += "echo Setting ${k}=${v}\n"
 653             // Write out the set itself to the script file`
 654             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 655         }
 656     }
 657
 658     return stepScript
 659 }
 660
 661 // Append an existing script to an environment script.
 662 // Returns string of commands to do this.
 663 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 664     assert (os == 'Windows_NT')
 665     def stepScript = ''
 666
 667     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 668     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 669
 670     return stepScript
 671 }
 672
 673 // Finalize an environment setting script.
 674 // Returns string of commands to do this.
 675 def static envScriptFinalize(def os, def stepScriptLocation) {
 676     def stepScript = ''
 677
 678     if (os == 'Windows_NT') {
 679         // Display the resulting script. This is useful when looking at the output log file.
 680         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 681         stepScript += "type ${stepScriptLocation}\r\n"
 682     }
 683     else {
 684         stepScript += "chmod +x ${stepScriptLocation}\n"
 685     }
 686
 687     return stepScript
 688 }
 689
 690 def static isNeedDocker(def architecture, def os, def isBuild) {
 691     if (isBuild) {
 692         if (architecture == 'x86' && os == 'Ubuntu') {
 693             return true
 694         }
 695         else if (architecture == 'arm') {
 696             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 697                 return true
 698             }
 699         }
 700     }
 701     else {
 702         if (architecture == 'x86' && os == 'Ubuntu') {
 703             return true
 704         }
 705     }
 706     return false
 707 }
 708
 709 def static getDockerImageName(def architecture, def os, def isBuild) {
 710     // We must change some docker private images to official later
 711     if (isBuild) {
 712         if (architecture == 'x86' && os == 'Ubuntu') {
 713             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 714         }
 715         else if (architecture == 'arm') {
 716             if (os == 'Ubuntu') {
 717                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 718             }
 719             else if (os == 'Ubuntu16.04') {
 720                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 721             }
 722             else if (os == 'Tizen') {
 723                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 724             }
 725         }
 726     }
 727     else {
 728         if (architecture == 'x86' && os == 'Ubuntu') {
 729             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 730         }
 731     }
 732     println("Unknown architecture to use docker: ${architecture} ${os}");
 733     assert false
 734 }
 735
 736 // Calculates the name of the build job based on some typical parameters.
 737 //
 738 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 739     // If the architecture is x64, do not add that info into the build name.
 740     // Need to change around some systems and other builds to pick up the right builds
 741     // to do that.
 742
 743     def suffix = scenario != 'default' ? "_${scenario}" : '';
 744     if (isBuildOnly) {
 745         suffix += '_bld'
 746     }
 747     def baseName = ''
 748     switch (architecture) {
 749         case 'x64':
 750             if (scenario == 'default') {
 751                 // For now we leave x64 off of the name for compatibility with other jobs
 752                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 753             }
 754             else if (scenario == 'formatting') {
 755                 // we don't care about the configuration for the formatting job. It runs all configs
 756                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 757             }
 758             else {
 759                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 760             }
 761             break
 762         case 'arm64':
 763             if (os.toLowerCase() == "windows_nt") {
 764                 // These are cross builds
 765                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 766             }
 767             else {
 768                 // Defaults to a small page size set of machines.
 769                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 770             }
 771             break
 772         case 'arm':
 773             // These are cross builds
 774             if (os == 'Tizen') {
 775                 // ABI: softfp
 776                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 777             }
 778             else {
 779                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 780             }
 781             break
 782         case 'armlb':
 783             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 784             break
 785         case 'x86':
 786         case 'x86_arm_altjit':
 787         case 'x64_arm64_altjit':
 788             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 789             break
 790         default:
 791             println("Unknown architecture: ${architecture}");
 792             assert false
 793             break
 794     }
 795
 796     return baseName + suffix
 797 }
 798
 799 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 800
 801     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 802     if (os == 'Windows_NT') {
 803         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 804             if (!(branch in Constants.WindowsArm64Branches)) {
 805                 return
 806             }
 807         }
 808     }
 809
 810     // Check scenario.
 811     switch (scenario) {
 812         case 'default':
 813             switch (architecture) {
 814                 case 'x64':
 815                 case 'x86':
 816                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 817                         addPeriodicTriggerHelper(job, '@daily')
 818                     }
 819                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 820                         addGithubPushTriggerHelper(job)
 821                     }
 822                     break
 823                 case 'arm':
 824                 case 'armlb':
 825                 case 'x86_arm_altjit':
 826                 case 'x64_arm64_altjit':
 827                     addGithubPushTriggerHelper(job)
 828                     break
 829                 case 'arm64':
 830                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 831                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 832                     break
 833                 default:
 834                     println("Unknown architecture: ${architecture}");
 835                     assert false
 836                     break
 837             }
 838             break
 839         case 'r2r':
 840             assert !(os in bidailyCrossList)
 841             // r2r gets a push trigger for checked/release
 842             if (configuration == 'Checked' || configuration == 'Release') {
 843                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 844                 if (architecture == 'x64' && os != 'OSX10.12') {
 845                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 846                     if (isFlowJob || os == 'Windows_NT') {
 847                         addGithubPushTriggerHelper(job)
 848                     }
 849                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 850                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 851                     if (isFlowJob) {
 852                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 853                     }
 854                 }
 855                 // For x86, only add per-commit jobs for Windows
 856                 else if (architecture == 'x86') {
 857                     if (os == 'Windows_NT') {
 858                         addGithubPushTriggerHelper(job)
 859                     }
 860                 }
 861                 // arm64 r2r jobs should only run daily.
 862                 else if (architecture == 'arm64') {
 863                     if (os == 'Windows_NT') {
 864                         addPeriodicTriggerHelper(job, '@daily')
 865                     }
 866                 }
 867             }
 868             break
 869         case 'r2r_jitstress1':
 870         case 'r2r_jitstress2':
 871         case 'r2r_jitstressregs1':
 872         case 'r2r_jitstressregs2':
 873         case 'r2r_jitstressregs3':
 874         case 'r2r_jitstressregs4':
 875         case 'r2r_jitstressregs8':
 876         case 'r2r_jitstressregs0x10':
 877         case 'r2r_jitstressregs0x80':
 878         case 'r2r_jitstressregs0x1000':
 879         case 'r2r_jitminopts':
 880         case 'r2r_jitforcerelocs':
 881         case 'r2r_gcstress15':
 882             assert !(os in bidailyCrossList)
 883
 884             // GCStress=C is currently not supported on OS X
 885             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 886                 break
 887             }
 888
 889             // GC Stress 15 r2r gets a push trigger for checked/release
 890             if (configuration == 'Checked' || configuration == 'Release') {
 891                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 892                 if (architecture == 'x64') {
 893                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 894                     if (isFlowJob || os == 'Windows_NT') {
 895                         // Add a weekly periodic trigger
 896                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 897                     }
 898                 }
 899                 // For x86, only add per-commit jobs for Windows
 900                 else if (architecture == 'x86') {
 901                     if (os == 'Windows_NT') {
 902                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 903                     }
 904                 }
 905             }
 906             break
 907         case 'longgc':
 908             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 909             assert configuration == 'Release'
 910             assert architecture == 'x64'
 911             addPeriodicTriggerHelper(job, '@daily')
 912             // TODO: Add once external email sending is available again
 913             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 914             break
 915         case 'gcsimulator':
 916             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 917             assert configuration == 'Release'
 918             assert architecture == 'x64'
 919             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 920             // TODO: Add once external email sending is available again
 921             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 922             break
 923         case 'standalone_gc':
 924             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 925             assert (configuration == 'Release' || configuration == 'Checked')
 926             // TODO: Add once external email sending is available again
 927             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 928             addPeriodicTriggerHelper(job, '@daily')
 929             break
 930         case 'gc_reliability_framework':
 931             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 932             assert (configuration == 'Release' || configuration == 'Checked')
 933             // Only triggered by phrase.
 934             break
 935         case 'ilrt':
 936             assert !(os in bidailyCrossList)
 937             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 938             if (architecture == 'x64' && configuration == 'Release') {
 939                 // We don't expect to see a job generated except in these scenarios
 940                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 941                 if (isFlowJob || os == 'Windows_NT') {
 942                     addPeriodicTriggerHelper(job, '@daily')
 943                 }
 944             }
 945             break
 946         case 'jitdiff':
 947             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 948             assert configuration == 'Checked'
 949             assert (architecture == 'x64' || architecture == 'x86')
 950             addGithubPushTriggerHelper(job)
 951             break
 952         case 'formatting':
 953             assert (os == 'Windows_NT' || os == "Ubuntu")
 954             assert architecture == 'x64'
 955             addGithubPushTriggerHelper(job)
 956             break
 957         case 'jitstressregs1':
 958         case 'jitstressregs2':
 959         case 'jitstressregs3':
 960         case 'jitstressregs4':
 961         case 'jitstressregs8':
 962         case 'jitstressregs0x10':
 963         case 'jitstressregs0x80':
 964         case 'jitstressregs0x1000':
 965         case 'minopts':
 966         case 'forcerelocs':
 967         case 'jitstress1':
 968         case 'jitstress2':
 969         case 'jitstress2_jitstressregs1':
 970         case 'jitstress2_jitstressregs2':
 971         case 'jitstress2_jitstressregs3':
 972         case 'jitstress2_jitstressregs4':
 973         case 'jitstress2_jitstressregs8':
 974         case 'jitstress2_jitstressregs0x10':
 975         case 'jitstress2_jitstressregs0x80':
 976         case 'jitstress2_jitstressregs0x1000':
 977         case 'tailcallstress':
 978         case 'jitsse2only':
 979         case 'jitnosimd':
 980         case 'corefx_baseline':
 981         case 'corefx_minopts':
 982         case 'corefx_jitstress1':
 983         case 'corefx_jitstress2':
 984         case 'corefx_jitstressregs1':
 985         case 'corefx_jitstressregs2':
 986         case 'corefx_jitstressregs3':
 987         case 'corefx_jitstressregs4':
 988         case 'corefx_jitstressregs8':
 989         case 'corefx_jitstressregs0x10':
 990         case 'corefx_jitstressregs0x80':
 991         case 'corefx_jitstressregs0x1000':
 992         case 'zapdisable':
 993             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 994                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 995                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 996                     if (os == 'Windows_NT') {
 997                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 998                         addPeriodicTriggerHelper(job, '@weekly')
 999                     }
1000                 }
1001                 else {
1002                     addPeriodicTriggerHelper(job, '@daily')
1003                 }
1004             }
1005             break
1006         case 'heapverify1':
1007         case 'gcstress0x3':
1008             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1009                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1010                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1011                     if (os == 'Windows_NT') {
1012                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1013                         addPeriodicTriggerHelper(job, '@weekly')
1014                     }
1015                     // TODO: Add once external email sending is available again
1016                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1017                 }
1018                 else {
1019                     addPeriodicTriggerHelper(job, '@weekly')
1020                 }
1021             }
1022             break
1023         case 'gcstress0xc':
1024         case 'gcstress0xc_zapdisable':
1025         case 'gcstress0xc_zapdisable_jitstress2':
1026         case 'gcstress0xc_zapdisable_heapverify1':
1027         case 'gcstress0xc_jitstress1':
1028         case 'gcstress0xc_jitstress2':
1029         case 'gcstress0xc_minopts_heapverify1':
1030             // GCStress=C is currently not supported on OS X
1031             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1032                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1033                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1034                     if (os == 'Windows_NT') {
1035                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1036                         addPeriodicTriggerHelper(job, '@weekly')
1037                     }
1038                     // TODO: Add once external email sending is available again
1039                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1040                 }
1041                 else {
1042                     addPeriodicTriggerHelper(job, '@weekly')
1043                 }
1044             }
1045             break
1046
1047         case 'illink':
1048             // Testing on other operating systems TBD
1049             assert (os == 'Windows_NT' || os == 'Ubuntu')
1050             if (architecture == 'x64' || architecture == 'x86') {
1051                 if (configuration == 'Checked') {
1052                     addPeriodicTriggerHelper(job, '@daily')
1053                 }
1054             }
1055             break
1056
1057         case 'tieredcompilation':
1058         case 'corefx_tieredcompilation':
1059             // No periodic jobs just yet, still testing
1060             break
1061
1062         default:
1063             println("Unknown scenario: ${scenario}");
1064             assert false
1065             break
1066     }
1067     return
1068 }
1069
1070 // **************************
1071 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1072 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1073 // builds will do a couple extra steps.
1074 // **************************
1075
1076 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1077 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1078 // it's just used for internal builds
1079 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1080 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1081     if (isWindowsBuildOnlyJob) {
1082         return
1083     }
1084
1085     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1086     // Non pull request builds.
1087     if (!isPR) {
1088         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1089         return
1090     }
1091
1092      def arm64Users = [
1093         'AndyAyersMS',
1094         'briansull',
1095         'BruceForstall',
1096         'CarolEidt',
1097         'cmckinsey',
1098         'erozenfeld',
1099         'janvorli',
1100         'jashook',
1101         'JosephTremoulet',
1102         'pgodeq',
1103         'pgavlin',
1104         'rartemev',
1105         'russellhadley',
1106         'RussKeldorph',
1107         'sandreenko',
1108         'sdmaclea',
1109         'swaroop-sridhar',
1110         'jkotas',
1111         'markwilkie',
1112         'weshaggard'
1113     ]
1114
1115     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1116     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1117     def osGroup = getOSGroup(os)
1118     switch (architecture) {
1119         case 'x64': // editor brace matching: {
1120             if (scenario == 'formatting') {
1121                 assert configuration == 'Checked'
1122                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1123                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1124                 }
1125                 break
1126             }
1127
1128             switch (os) {
1129                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1130                 case 'Debian8.4':
1131                 case 'RHEL7.2':
1132                     if (scenario == 'default') {
1133                         assert !isFlowJob
1134                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1135                     }
1136                     break
1137                 case 'Ubuntu16.04':
1138                     assert !isFlowJob
1139                     assert scenario == 'default'
1140                     // Distinguish with the other architectures (arm and x86)
1141                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1142                     break
1143                 case 'Fedora24':
1144                 case 'Ubuntu16.10':
1145                     assert !isFlowJob
1146                     assert scenario == 'default'
1147                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1148                     break
1149                 case 'Ubuntu':
1150                     if (scenario == 'illink') {
1151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1152                         break
1153                     }
1154                     // fall through
1155                 case 'OSX10.12':
1156                     // Triggers on the non-flow jobs aren't necessary here
1157                     // Corefx testing uses non-flow jobs.
1158                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1159                         break
1160                     }
1161                     switch (scenario) {
1162                         case 'default':
1163                             // OSX uses checked for default PR tests
1164                             if (configuration == 'Checked') {
1165                                 // Default trigger
1166                                 assert !job.name.contains("centos")
1167                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1168                             }
1169                             break
1170                         case 'jitdiff':
1171                             if (configuration == 'Checked') {
1172                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1173                             }
1174                             break
1175                         case 'ilrt':
1176                             if (configuration == 'Release') {
1177                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1178                             }
1179                             break
1180                         case 'longgc':
1181                             if (configuration == 'Release') {
1182                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1183                             }
1184                             break
1185                         case 'gcsimulator':
1186                             if (configuration == 'Release') {
1187                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1188                             }
1189                             break
1190                         case 'standalone_gc':
1191                             if (configuration == 'Release' || configuration == 'Checked') {
1192                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1193                             }
1194                             break
1195                         case 'gc_reliability_framework':
1196                             if (configuration == 'Release' || configuration == 'Checked') {
1197                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1198                             }
1199                             break
1200                         default:
1201                             if (isJitStressScenario(scenario)) {
1202                                 def displayStr = getStressModeDisplayName(scenario)
1203                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1204                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1205                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1206                             }
1207                             else if (isR2RScenario(scenario)) {
1208                                 if (configuration == 'Release' || configuration == 'Checked') {
1209                                     def displayStr = getR2RDisplayName(scenario)
1210                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1211                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1212                                 }
1213                             }
1214                             else {
1215                                 println("Unknown scenario: ${scenario}");
1216                                 assert false
1217                             }
1218                             break
1219                     }
1220                     break
1221
1222                 case 'CentOS7.1':
1223                     switch (scenario) {
1224                         case 'default':
1225                             // CentOS uses checked for default PR tests while debug is build only
1226                             if (configuration == 'Debug') {
1227                                 // Default trigger
1228                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1229                             }
1230
1231                             // Make sure this is a flow job to get build and test.
1232                             if (configuration == 'Checked' && isFlowJob) {
1233                                 assert job.name.contains("flow")
1234                                 // Default trigger
1235                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1236                             }
1237                             break
1238                         default:
1239                             if (isR2RScenario(scenario)) {
1240                                 if (configuration == 'Release' || configuration == 'Checked') {
1241                                     def displayStr = getR2RDisplayName(scenario)
1242                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1243                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1244                                 }
1245                             }
1246                             break
1247                     }
1248                     break
1249
1250                 case 'Windows_NT':
1251                     switch (scenario) {
1252                         case 'default':
1253                             // Default trigger
1254                             if (configuration == 'Checked') {
1255                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1256                             }
1257                             break
1258                         case 'jitdiff':
1259                             if (configuration == 'Checked') {
1260                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1261                             }
1262                             break
1263                         case 'ilrt':
1264                             if (configuration == 'Release') {
1265                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1266                             }
1267                             break
1268                         case 'longgc':
1269                             if (configuration == 'Release') {
1270                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1271                             }
1272                             break
1273                         case 'gcsimulator':
1274                             if (configuration == 'Release') {
1275                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1276                             }
1277                             break
1278                         case 'standalone_gc':
1279                             if (configuration == 'Release' || configuration == 'Checked') {
1280                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1281                             }
1282                             break
1283                         case 'gc_reliability_framework':
1284                             if (configuration == 'Release' || configuration == 'Checked') {
1285                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1286                             }
1287                             break
1288                         case 'illink':
1289                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1290                             break
1291                         default:
1292                             if (isJitStressScenario(scenario)) {
1293                                 def displayStr = getStressModeDisplayName(scenario)
1294                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1295                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1296                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1297                             }
1298                             else if (isR2RScenario(scenario)) {
1299                                 if (configuration == 'Release' || configuration == 'Checked') {
1300                                     def displayStr = getR2RDisplayName(scenario)
1301                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1302                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1303                                 }
1304                             }
1305                             else {
1306                                 println("Unknown scenario: ${scenario}");
1307                                 assert false
1308                             }
1309                             break
1310                     }
1311                     break
1312                 default:
1313                     println("Unknown os: ${os}");
1314                     assert false
1315                     break
1316             }
1317             break
1318         // editor brace matching: }
1319         case 'armlb':
1320         case 'arm': // editor brace matching: {
1321             switch (os) {
1322                 case 'Ubuntu':
1323                 case 'Ubuntu16.04':
1324                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1325                         break
1326                     }
1327                     assert scenario == 'default'
1328                     job.with {
1329                         publishers {
1330                             azureVMAgentPostBuildAction {
1331                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1332                             }
1333                         }
1334                     }
1335                     if (configuration == 'Debug') {
1336                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1337                     }
1338                     else {
1339                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1340                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1341                     }
1342                     break
1343                 case 'Tizen':
1344                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1345                         break
1346                     }
1347                     architecture='armel'
1348                     job.with {
1349                         publishers {
1350                             azureVMAgentPostBuildAction {
1351                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1352                             }
1353                         }
1354                     }
1355                     if (configuration == 'Checked') {
1356                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1357                     }
1358                     else {
1359                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1360                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1361                     }
1362                     break
1363                 case 'Windows_NT':
1364                     // Triggers on the non-flow jobs aren't necessary here
1365                     if (!isFlowJob) {
1366                         break
1367                     }
1368
1369                     // Set up a private trigger
1370                     def contextString = "${os} ${architecture} Cross ${configuration}"
1371                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1372                     if (scenario == 'default') {
1373                         contextString += " Innerloop"
1374                         triggerString += "\\W+Innerloop"
1375                     }
1376                     else {
1377                         contextString += " ${scenario}"
1378                         triggerString += "\\W+${scenario}"
1379                     }
1380
1381                     if (configuration == 'Debug') {
1382                         contextString += " Build"
1383                         triggerString += "\\W+Build"
1384                     } else {
1385                         contextString += " Build and Test"
1386                         triggerString += "\\W+Build and Test"
1387                     }
1388
1389                     triggerString += ".*"
1390
1391                     switch (scenario) {
1392                         case 'default':
1393                             // Only Checked is a default trigger.
1394                             if (configuration == 'Checked')
1395                             {
1396                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1397                             }
1398                             else
1399                             {
1400                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1401                             }
1402                             break
1403                         default:
1404                             // Stress jobs will use this code path.
1405                             if (isArmWindowsScenario(scenario)) {
1406                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1407                             }
1408                             break
1409                     }
1410                     break
1411                 default:
1412                     println("NYI os: ${os}");
1413                     assert false
1414                     break
1415             }
1416             break
1417         // editor brace matching: }
1418         case 'arm64': // editor brace matching: {
1419             // Set up a private trigger
1420             def contextString = "${os} ${architecture} Cross ${configuration}"
1421             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1422             if (scenario == 'default') {
1423                 contextString += " Innerloop"
1424                 triggerString += "\\W+Innerloop"
1425             }
1426             else {
1427                 contextString += " ${scenario}"
1428                 triggerString += "\\W+${scenario}"
1429             }
1430
1431             if (configuration == 'Debug') {
1432                 contextString += " Build"
1433                 triggerString += "\\W+Build"
1434             } else {
1435                 contextString += " Build and Test"
1436                 triggerString += "\\W+Build and Test"
1437             }
1438
1439             triggerString += ".*"
1440
1441             switch (os) {
1442                 case 'Ubuntu':
1443                 case 'Ubuntu16.04':
1444                     switch (scenario) {
1445                         case 'default':
1446                             if (configuration == 'Debug' && !isFlowJob) {
1447                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1448                             }
1449                             else {
1450                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1451                             }
1452
1453                             break
1454                         default:
1455                             if (isR2RScenario(scenario)) {
1456                                 if (configuration == 'Checked' || configuration == 'Release') {
1457                                     def displayStr = getR2RDisplayName(scenario)
1458                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1459                                 }
1460                             }
1461                             break
1462                     }
1463                     break
1464                 case 'Windows_NT':
1465                     // Triggers on the non-flow jobs aren't necessary here
1466                     if (!isFlowJob) {
1467                         break
1468                     }
1469
1470                     assert isArmWindowsScenario(scenario)
1471                     switch (scenario) {
1472                         case 'default':
1473                             if (configuration == 'Checked') {
1474                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1475                             }
1476                             else {
1477                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1478                             }
1479                             break
1480                         default:
1481                             // Stress jobs will use this code path.
1482                             if (isArmWindowsScenario(scenario)) {
1483                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1484                             }
1485                             break
1486                     }
1487                     break
1488                 default:
1489                     println("NYI os: ${os}");
1490                     assert false
1491                     break
1492             }
1493             break
1494         // editor brace matching: }
1495         case 'x86': // editor brace matching: {
1496             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1497             if (os == 'Ubuntu') {
1498                 // Triggers on the non-flow jobs aren't necessary here
1499                 if (!isFlowJob) {
1500                     break
1501                 }
1502                 // on-demand only for ubuntu x86
1503                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1504                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1505                 break
1506             }
1507             switch (scenario) {
1508                 case 'default':
1509                     if (configuration == 'Checked') {
1510                         assert !job.name.contains("centos")
1511                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1512                     }
1513                     else {
1514                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1515                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1516                     }
1517                     break
1518                 case 'ilrt':
1519                     if (configuration == 'Release') {
1520                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1521                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1522                     }
1523                     break
1524                 case 'longgc':
1525                     if (configuration == 'Release') {
1526                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1527                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1528                     }
1529                     break
1530                 case 'gcsimulator':
1531                     if (configuration == 'Release') {
1532                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1533                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1534                     }
1535                     break
1536                 case 'standalone_gc':
1537                     if (configuration == 'Release' || configuration == 'Checked') {
1538                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1539                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1540                     }
1541                     break
1542                 case 'illink':
1543                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1544                     break
1545                 default:
1546                     if (isJitStressScenario(scenario)) {
1547                         def displayStr = getStressModeDisplayName(scenario)
1548                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1549                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1550                     }
1551                     else if (isR2RScenario(scenario)) {
1552                         if (configuration == 'Release' || configuration == 'Checked') {
1553                             def displayStr = getR2RDisplayName(scenario)
1554                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1555                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1556                         }
1557                     }
1558                     else {
1559                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1560                         assert false
1561                     }
1562                     break
1563             }
1564             break
1565          // editor brace matching: }
1566         case 'x64_arm64_altjit':
1567         case 'x86_arm_altjit': // editor brace matching: {
1568             assert (os == 'Windows_NT')
1569             switch (scenario) {
1570                 case 'default':
1571                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1572                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1573                     break
1574                 default:
1575                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1576                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1577                     break
1578             }
1579             break
1580         // editor brace matching: }
1581         default:
1582             println("Unknown architecture: ${architecture}");
1583             assert false
1584             break
1585     }
1586 }
1587
1588 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1589     def buildCommands = [];
1590     def osGroup = getOSGroup(os)
1591     def lowerConfiguration = configuration.toLowerCase()
1592
1593     def priority = '1'
1594     if (scenario == 'default' && isPR == true) {
1595         priority = '0'
1596     }
1597
1598     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1599
1600     def enableCorefxTesting = isCoreFxScenario(scenario)
1601
1602     // Calculate the build steps, archival, and xunit results
1603     switch (os) {
1604         case 'Windows_NT': // editor brace matching: {
1605             switch (architecture) {
1606                 case 'x64':
1607                 case 'x86':
1608                 case 'x86_arm_altjit':
1609                 case 'x64_arm64_altjit':
1610                     def arch = architecture
1611                     def buildOpts = ''
1612                     if (architecture == 'x86_arm_altjit') {
1613                         arch = 'x86'
1614                     }
1615                     else if (architecture == 'x64_arm64_altjit') {
1616                         arch = 'x64'
1617                     }
1618
1619                     if (scenario == 'formatting') {
1620                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1621                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1622                         break
1623                     }
1624
1625                     if (scenario == 'illink') {
1626                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1627                     }
1628
1629                     // If it is a release build for windows, ensure PGO is used, else fail the build
1630                     if ((lowerConfiguration == 'release') &&
1631                         (scenario in Constants.basicScenarios) &&
1632                         (architecture != 'x86_arm_altjit') &&
1633                         (architecture != 'x64_arm64_altjit')) {
1634
1635                         buildOpts += ' -enforcepgo'
1636                     }
1637
1638                     if (enableCorefxTesting) {
1639                         buildOpts += ' skiptests';
1640                     } else {
1641                         buildOpts += " -priority=${priority}"
1642                     }
1643
1644                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1645                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1646                     // 35 characters long.
1647
1648                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1649
1650                     if (!isBuildOnly) {
1651                         def runtestArguments = ''
1652                         def testOpts = 'collectdumps'
1653
1654                         if (isR2RScenario(scenario)) {
1655
1656                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1657                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1658                             // to cause the tests to be crossgen'ed.
1659
1660                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1661                                 testOpts += ' crossgenaltjit protononjit.dll'
1662                             } else {
1663                                 testOpts += ' crossgen'
1664                             }
1665
1666                             testOpts += ' runcrossgentests'
1667                         }
1668                         else if (scenario == 'jitdiff') {
1669                             testOpts += ' jitdisasm crossgen'
1670                         }
1671                         else if (scenario == 'ilrt') {
1672                             testOpts += ' ilasmroundtrip'
1673                         }
1674                         else if (isLongGc(scenario)) {
1675                             testOpts += " ${scenario} sequential"
1676                         }
1677                         else if (scenario == 'standalone_gc') {
1678                             testOpts += ' gcname clrgc.dll'
1679                         }
1680                         else if (scenario == 'illink') {
1681                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1682                         }
1683
1684                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1685                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1686                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1687                         // which is read by the xunit harness.)
1688                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1689                         {
1690                             def timeout = 1800000
1691                             testOpts += " timeout ${timeout}"
1692                         }
1693
1694                         // If we are running a stress mode, we should write out the set of key
1695                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1696
1697                         def envScriptPath = ''
1698                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1699                             def buildCommandsStr = ''
1700                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1701                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1702
1703                             if (isJitStressScenario(scenario)) {
1704                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1705                             }
1706                             else if (isR2RStressScenario(scenario)) {
1707                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1708                             }
1709
1710                             if (architecture == 'x86_arm_altjit') {
1711                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1712                             }
1713                             else if (architecture == 'x64_arm64_altjit') {
1714                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1715                             }
1716
1717                             envScriptFinalize(os, envScriptPath)
1718
1719                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1720                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1721                             // into a single script.
1722                             buildCommands += buildCommandsStr
1723                         }
1724                         else if (architecture == 'x86_arm_altjit') {
1725                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1726                         }
1727                         else if (architecture == 'x64_arm64_altjit') {
1728                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1729                         }
1730                         if (envScriptPath != '') {
1731                             testOpts += " TestEnv ${envScriptPath}"
1732                         }
1733
1734                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1735
1736                         if (enableCorefxTesting) {
1737                             def workspaceRelativeFxRoot = "_/fx"
1738                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1739
1740                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1741
1742                             // Archive and process (only) the test results
1743                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1744                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1745
1746                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1747                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1748                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1749                         }
1750                         else if (isGcReliabilityFramework(scenario)) {
1751                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1752                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1753                         }
1754                         else {
1755                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1756                         }
1757                     }
1758
1759                     if (!enableCorefxTesting) {
1760                         // Run the rest of the build
1761                         // Build the mscorlib for the other OS's
1762                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1763                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1764
1765                         if (arch == "x64") {
1766                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1767                         }
1768
1769                         // Zip up the tests directory so that we don't use so much space/time copying
1770                         // 10s of thousands of files around.
1771                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1772
1773                         if (!isJitStressScenario(scenario)) {
1774                             // For windows, pull full test results and test drops for x86/x64.
1775                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1776                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1777                         }
1778
1779                         if (scenario == 'jitdiff') {
1780                             // retrive jit-dasm output for base commit, and run jit-diff
1781                             if (!isBuildOnly) {
1782                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1783                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1784                             }
1785                         }
1786
1787                         if (!isBuildOnly) {
1788                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1789                         }
1790                     }
1791                     break
1792                 case 'armlb':
1793                 case 'arm':
1794                     assert isArmWindowsScenario(scenario)
1795
1796                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1797                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1798
1799                     def buildArchitecture = 'arm'
1800
1801                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1802                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1803                     def armCrossgenOpt = ''
1804                     if (architecture == 'armlb') {
1805                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1806                     }
1807
1808                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1809                     priority = '1'
1810
1811                     // This is now a build only job. Do not run tests. Use the flow job.
1812                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1813
1814                     // Zip up the tests directory so that we don't use so much space/time copying
1815                     // 10s of thousands of files around.
1816                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1817
1818                     // Add archival.
1819                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1820                     break
1821                 case 'arm64':
1822                     assert isArmWindowsScenario(scenario)
1823
1824                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1825                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1826
1827                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1828                     priority = '1'
1829
1830                     // This is now a build only job. Do not run tests. Use the flow job.
1831                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1832
1833                     // Zip up the tests directory so that we don't use so much space/time copying
1834                     // 10s of thousands of files around.
1835                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1836
1837                     // Add archival.
1838                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1839                     break
1840                 default:
1841                     println("Unknown architecture: ${architecture}");
1842                     assert false
1843                     break
1844             }
1845             break
1846         // editor brace matching: }
1847         case 'Ubuntu':
1848         case 'Ubuntu16.04':
1849         case 'Ubuntu16.10':
1850         case 'Debian8.4':
1851         case 'OSX10.12':
1852         case 'CentOS7.1':
1853         case 'RHEL7.2':
1854         case 'Tizen':
1855         case 'Fedora24': // editor brace matching: {
1856             switch (architecture) {
1857                 case 'x64':
1858                 case 'x86':
1859                     if (architecture == 'x86' && os == 'Ubuntu') {
1860                         // build and PAL test
1861                         def dockerImage = getDockerImageName(architecture, os, true)
1862                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1863                         dockerImage = getDockerImageName(architecture, os, false)
1864                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1865                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1866                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1867                         break
1868                     }
1869
1870                     if (scenario == 'formatting') {
1871                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1872                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1873                         break
1874                     }
1875
1876                     if (scenario == 'illink') {
1877                         assert(os == 'Ubuntu')
1878                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1879                     }
1880
1881                     if (!enableCorefxTesting) {
1882                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1883                         // only on supported OS platforms.
1884                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1885                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1886
1887                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1888                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1889
1890                         // Basic archiving of the build
1891                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1892                         // And pal tests
1893                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1894                     }
1895                     else {
1896                         // Corefx stress testing
1897                         assert os == 'Ubuntu'
1898                         assert architecture == 'x64'
1899                         assert lowerConfiguration == 'checked'
1900                         assert isJitStressScenario(scenario)
1901
1902                         // Build coreclr
1903                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1904
1905                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1906
1907                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1908                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1909                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1910                         buildCommands += envScriptCmds
1911
1912                         // Build and text corefx
1913                         def workspaceRelativeFxRoot = "_/fx"
1914                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1915
1916                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1917
1918                         // Archive and process (only) the test results
1919                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1920                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1921                     }
1922                     break
1923                 case 'arm64':
1924                     if (!enableCorefxTesting) {
1925                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1926
1927                         // HACK -- Arm64 does not have corefx jobs yet.
1928                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1929                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1930                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1931                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1932
1933                         // Basic archiving of the build
1934                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1935                     }
1936                     break
1937                 case 'arm':
1938                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1939                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1940
1941                     // default values for Ubuntu
1942                     def arm_abi="arm"
1943                     def linuxCodeName="trusty"
1944                     if (os == 'Ubuntu16.04') {
1945                         linuxCodeName="xenial"
1946                     }
1947                     else if (os == 'Tizen') {
1948                         arm_abi="armel"
1949                         linuxCodeName="tizen"
1950                     }
1951
1952                     // Unzip the Windows test binaries first. Exit with 0
1953                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1954
1955                     // Unpack the corefx binaries
1956                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1957                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1958                     if (os != 'Tizen') {
1959                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1960                     }
1961                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1962                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1963                     if (os == 'Ubuntu') {
1964                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1965                     }
1966
1967                     // Call the ARM CI script to cross build and test using docker
1968                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1969                     --mode=docker \\
1970                     --${arm_abi} \\
1971                     --linuxCodeName=${linuxCodeName} \\
1972                     --buildConfig=${lowerConfiguration} \\
1973                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1974                     --coreFxBinDir=./bin/CoreFxBinDir \\
1975                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1976
1977                     // Basic archiving of the build, no pal tests
1978                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1979                     break
1980                 default:
1981                     println("Unknown architecture: ${architecture}");
1982                     assert false
1983                     break
1984             }
1985             break
1986         // editor brace matching: }
1987         default:
1988             println("Unknown os: ${os}");
1989             assert false
1990             break
1991     } // os
1992
1993     return buildCommands
1994 }
1995
1996 Constants.allScenarios.each { scenario ->
1997     [true, false].each { isPR ->
1998         Constants.architectureList.each { architecture ->
1999             Constants.configurationList.each { configuration ->
2000                 Constants.osList.each { os ->
2001                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2002                     // and reset the os to Windows_NT
2003                     def isBuildOnly = false
2004                     if (os == 'Windows_NT_BuildOnly') {
2005                         isBuildOnly = true
2006                         os = 'Windows_NT'
2007                     }
2008
2009                     // Tizen is only supported for arm architecture
2010                     if (os == 'Tizen' && architecture != 'arm') {
2011                         return
2012                     }
2013
2014                     // Skip totally unimplemented (in CI) configurations.
2015                     switch (architecture) {
2016                         case 'arm64':
2017                             if (os == 'Ubuntu16.04') {
2018                                 os = 'Ubuntu'
2019                             }
2020
2021                             // Windows and Ubuntu only
2022                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2023                                 return
2024                             }
2025                             break
2026                         case 'arm':
2027                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2028                                 return
2029                             }
2030                             break
2031                         case 'armlb':
2032                             if (os != 'Windows_NT') {
2033                                 return
2034                             }
2035                             break
2036                         case 'x86':
2037                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2038                                 return
2039                             }
2040                             break
2041                         case 'x86_arm_altjit':
2042                         case 'x64_arm64_altjit':
2043                             if (os != 'Windows_NT') {
2044                                 return
2045                             }
2046                             break
2047                         case 'x64':
2048                             // Everything implemented
2049                             break
2050                         default:
2051                             println("Unknown architecture: ${architecture}")
2052                             assert false
2053                             break
2054                     }
2055
2056                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2057                     // with checked builds)
2058                     if (isJitStressScenario(scenario)) {
2059                         if (configuration != 'Checked') {
2060                             return
2061                         }
2062
2063                         // Since these are just execution time differences,
2064                         // skip platforms that don't execute the tests here (Windows_NT only)
2065                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2066                         if (!isEnabledOS || isBuildOnly) {
2067                             return
2068                         }
2069
2070                         switch (architecture) {
2071                             case 'x64':
2072                             case 'x86':
2073                             case 'x86_arm_altjit':
2074                             case 'x64_arm64_altjit':
2075                                 // x86 ubuntu: default only
2076                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2077                                     return
2078                                 }
2079                                 // Windows: Everything implemented
2080                                 break
2081
2082                             default:
2083                                 // arm, arm64, armlb: stress is handled through flow jobs.
2084                                 return
2085                         }
2086                     }
2087                     else if (isR2RScenario(scenario)) {
2088                         if (os != 'Windows_NT') {
2089                             return
2090                         }
2091                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2092                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2093                             return
2094                         }
2095                     }
2096                     else {
2097                         // Skip scenarios
2098                         switch (scenario) {
2099                             case 'ilrt':
2100                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2101                                 // the default scenario build
2102                                 if (os != 'Windows_NT') {
2103                                     return
2104                                 }
2105                                 // Only x64 for now
2106                                 if (architecture != 'x64') {
2107                                     return
2108                                 }
2109                                 // Release only
2110                                 if (configuration != 'Release') {
2111                                     return
2112                                 }
2113                                 break
2114                             case 'jitdiff':
2115                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2116                                     return
2117                                 }
2118                                 if (architecture != 'x64') {
2119                                     return
2120                                 }
2121                                 if (configuration != 'Checked') {
2122                                     return
2123                                 }
2124                                 break
2125                             case 'longgc':
2126                             case 'gcsimulator':
2127                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2128                                     return
2129                                 }
2130                                 if (architecture != 'x64') {
2131                                     return
2132                                 }
2133                                 if (configuration != 'Release') {
2134                                     return
2135                                 }
2136                                 break
2137                             case 'gc_reliability_framework':
2138                             case 'standalone_gc':
2139                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2140                                     return
2141                                 }
2142
2143                                 if (architecture != 'x64') {
2144                                     return
2145                                 }
2146
2147                                 if (configuration != 'Release' && configuration != 'Checked') {
2148                                     return
2149                                 }
2150                                 break
2151                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2152                             case 'formatting':
2153                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2154                                     return
2155                                 }
2156                                 if (architecture != 'x64') {
2157                                     return
2158                                 }
2159                                 if (configuration != 'Checked') {
2160                                     return
2161                                 }
2162                                 if (isBuildOnly) {
2163                                     return
2164                                 }
2165                                 break
2166                             case 'illink':
2167                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2168                                     return
2169                                 }
2170                                 if (architecture != 'x64' && architecture != 'x86') {
2171                                     return
2172                                 }
2173                                 if (isBuildOnly) {
2174                                     return
2175                                 }
2176                                 break
2177                             case 'default':
2178                                 // Nothing skipped
2179                                 break
2180                             default:
2181                                 println("Unknown scenario: ${scenario}")
2182                                 assert false
2183                                 break
2184                         }
2185                     }
2186
2187                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2188                     // runtime behavior, not compile-time behavior, are not interesting.
2189                     switch (architecture) {
2190                         case 'x86_arm_altjit':
2191                         case 'x64_arm64_altjit':
2192                             if (isGCStressRelatedTesting(scenario)) {
2193                                 return
2194                             }
2195                             break
2196                         default:
2197                             break
2198                     }
2199
2200                     // Calculate names
2201                     def lowerConfiguration = configuration.toLowerCase()
2202                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2203                     def folderName = getJobFolder(scenario)
2204
2205                     // Create the new job
2206                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2207                     addToViews(newJob, isPR, architecture, os)
2208
2209                     def machineAffinityOptions = null
2210
2211                     if (os != 'Windows_NT') {
2212                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2213                     }
2214                     else {
2215                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2216                     }
2217
2218                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2219
2220                     // Add all the standard options
2221                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2222                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2223
2224                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2225                     def osGroup = getOSGroup(os)
2226
2227                     newJob.with {
2228                         steps {
2229                             if (os == 'Windows_NT') {
2230                                 buildCommands.each { buildCommand ->
2231                                     batchFile(buildCommand)
2232                                 }
2233                             }
2234                             else {
2235                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2236                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2237                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2238                                     // Define the Windows Tests and Corefx build job names
2239                                     def WindowsTestsName = projectFolder + '/' +
2240                                                            Utilities.getFullJobName(project,
2241                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2242                                                                                     false)
2243                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2244                                                        Utilities.getFolderName(branch)
2245
2246                                     // Copy the Windows test binaries and the Corefx build binaries
2247                                     copyArtifacts(WindowsTestsName) {
2248                                         includePatterns('bin/tests/tests.zip')
2249                                         buildSelector {
2250                                             latestSuccessful(true)
2251                                         }
2252                                     }
2253
2254                                     def arm_abi = 'arm'
2255                                     def corefx_os = 'linux'
2256                                     if (os == 'Tizen') {
2257                                         arm_abi = 'armel'
2258                                         corefx_os = 'tizen'
2259                                     }
2260
2261                                     // Let's use release CoreFX to test checked CoreCLR,
2262                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2263                                     def corefx_lowerConfiguration = lowerConfiguration
2264                                     if ( lowerConfiguration == 'checked' ) {
2265                                         corefx_lowerConfiguration='release'
2266                                     }
2267
2268                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2269                                         includePatterns('bin/build.tar.gz')
2270                                         buildSelector {
2271                                             latestSuccessful(true)
2272                                         }
2273                                     }
2274                                 }
2275
2276                                 buildCommands.each { buildCommand ->
2277                                     shell(buildCommand)
2278                                 }
2279                             }
2280                         }
2281                     } // newJob.with
2282
2283                 } // os
2284             } // configuration
2285         } // architecture
2286     } // isPR
2287 } // scenario
2288
2289
2290 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2291 Constants.allScenarios.each { scenario ->
2292     [true, false].each { isPR ->
2293         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2294             Constants.crossList.each { os ->
2295                 if (architecture == 'arm64') {
2296                     if (os != "Ubuntu" && os != "Windows_NT") {
2297                         return
2298                     }
2299                 } else if (architecture == 'arm' || architecture == 'armlb') {
2300                     if (os != 'Windows_NT') {
2301                         return
2302                     }
2303                 }
2304                 else if (architecture == 'x86') {
2305                     if (os != "Ubuntu") {
2306                         return
2307                     }
2308                 }
2309
2310                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2311
2312                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2313                     return
2314                 }
2315
2316                 Constants.configurationList.each { configuration ->
2317
2318                     // First, filter based on OS.
2319
2320                     if (os == 'Windows_NT') {
2321                         if (!isArmWindowsScenario(scenario)) {
2322                             return
2323                         }
2324                     }
2325                     else {
2326                         // Non-Windows
2327                         if (architecture == 'arm64') {
2328                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2329                                 return
2330                             }
2331                         }
2332                         else if (architecture == 'x86') {
2333                             // Linux/x86 only want default test
2334                             if (scenario != 'default') {
2335                                 return
2336                             }
2337                         }
2338                     }
2339
2340                     // For CentOS, we only want Checked/Release builds.
2341                     if (os == 'CentOS7.1') {
2342                         if (configuration != 'Checked' && configuration != 'Release') {
2343                             return
2344                         }
2345                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2346                             return
2347                         }
2348                     }
2349
2350                     // For RedHat and Debian, we only do Release builds.
2351                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2352                         if (configuration != 'Release') {
2353                             return
2354                         }
2355                         if (scenario != 'default') {
2356                             return
2357                         }
2358                     }
2359
2360                     // Next, filter based on scenario.
2361
2362                     if (isJitStressScenario(scenario)) {
2363                         if (configuration != 'Checked') {
2364                             return
2365                         }
2366                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2367                         if (isCoreFxScenario(scenario)) {
2368                             return
2369                         }
2370                     }
2371                     else if (isR2RBaselineScenario(scenario)) {
2372                         if (configuration != 'Checked' && configuration != 'Release') {
2373                             return
2374                         }
2375                     }
2376                     else if (isR2RStressScenario(scenario)) {
2377                         if (configuration != 'Checked') {
2378                             return
2379                         }
2380                     }
2381                     else {
2382                         // Skip scenarios
2383                         switch (scenario) {
2384                             case 'ilrt':
2385                             case 'longgc':
2386                             case 'gcsimulator':
2387                                 // Long GC tests take a long time on non-Release builds
2388                                 // ilrt is also Release only
2389                                 if (configuration != 'Release') {
2390                                     return
2391                                 }
2392                                 break
2393                             case 'jitdiff':
2394                                 if (configuration != 'Checked') {
2395                                     return;
2396                                 }
2397                                 break
2398                             case 'gc_reliability_framework':
2399                             case 'standalone_gc':
2400                                 if (configuration != 'Release' && configuration != 'Checked') {
2401                                     return
2402                                 }
2403                                 break
2404                             case 'formatting':
2405                                 return
2406                             case 'illink':
2407                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2408                                     return
2409                                 }
2410                                 break
2411                             case 'default':
2412                                 // Nothing skipped
2413                                 break
2414                             default:
2415                                 println("Unknown scenario: ${scenario}")
2416                                 assert false
2417                                 break
2418                         }
2419                     }
2420
2421                     // Done filtering. Now, create the jobs.
2422
2423                     def lowerConfiguration = configuration.toLowerCase()
2424                     def osGroup = getOSGroup(os)
2425                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2426
2427                     def inputCoreCLRBuildName = projectFolder + '/' +
2428                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2429
2430                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2431                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2432                     def testBuildScenario = scenario
2433                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2434                         testBuildScenario = 'default'
2435                     }
2436
2437                     def inputWindowsTestBuildArch = architecture
2438                     if (architecture == "arm64" && os != "Windows_NT") {
2439                         // Use the x64 test build for arm64 unix
2440                         inputWindowsTestBuildArch = "x64"
2441                     }
2442
2443                     def inputWindowsTestsBuildName = ""
2444
2445                     if (isJitStressScenario(scenario)) {
2446                         inputWindowsTestsBuildName = projectFolder + '/' +
2447                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2448                     } else {
2449                         inputWindowsTestsBuildName = projectFolder + '/' +
2450                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2451
2452                     }
2453
2454
2455                     // Enable Server GC for Ubuntu PR builds
2456                     def serverGCString = ''
2457                     if (os == 'Ubuntu' && isPR) {
2458                         serverGCString = '--useServerGC'
2459                     }
2460
2461                     def testOpts = ''
2462
2463                     if (isR2RScenario(scenario)) {
2464
2465                         testOpts += ' --crossgen --runcrossgentests'
2466
2467                         if (scenario == 'r2r_jitstress1') {
2468                             testOpts += ' --jitstress=1'
2469                         }
2470                         else if (scenario == 'r2r_jitstress2') {
2471                             testOpts += ' --jitstress=2'
2472                         }
2473                         else if (scenario == 'r2r_jitstressregs1') {
2474                             testOpts += ' --jitstressregs=1'
2475                         }
2476                         else if (scenario == 'r2r_jitstressregs2') {
2477                             testOpts += ' --jitstressregs=2'
2478                         }
2479                         else if (scenario == 'r2r_jitstressregs3') {
2480                             testOpts += ' --jitstressregs=3'
2481                         }
2482                         else if (scenario == 'r2r_jitstressregs4') {
2483                             testOpts += ' --jitstressregs=4'
2484                         }
2485                         else if (scenario == 'r2r_jitstressregs8') {
2486                             testOpts += ' --jitstressregs=8'
2487                         }
2488                         else if (scenario == 'r2r_jitstressregs0x10') {
2489                             testOpts += ' --jitstressregs=0x10'
2490                         }
2491                         else if (scenario == 'r2r_jitstressregs0x80') {
2492                             testOpts += ' --jitstressregs=0x80'
2493                         }
2494                         else if (scenario == 'r2r_jitstressregs0x1000') {
2495                             testOpts += ' --jitstressregs=0x1000'
2496                         }
2497                         else if (scenario == 'r2r_jitminopts') {
2498                             testOpts += ' --jitminopts'
2499                         }
2500                         else if (scenario == 'r2r_jitforcerelocs') {
2501                             testOpts += ' --jitforcerelocs'
2502                         }
2503                         else if (scenario == 'r2r_gcstress15') {
2504                             testOpts += ' --gcstresslevel=0xF'
2505                         }
2506                     }
2507                     else if (scenario == 'jitdiff') {
2508                         testOpts += ' --jitdisasm --crossgen'
2509                     }
2510                     else if (scenario == 'illink') {
2511                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2512                     }
2513                     else if (isLongGc(scenario)) {
2514                         // Long GC tests behave very poorly when they are not
2515                         // the only test running (many of them allocate until OOM).
2516                         testOpts += ' --sequential'
2517
2518                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2519                         // for running long GC and GCSimulator tests, respectively. We don't use them
2520                         // here because using a playlist file produces much more readable output on the CI machines
2521                         // and reduces running time.
2522                         //
2523                         // The Long GC playlist contains all of the tests that are
2524                         // going to be run. The GCSimulator playlist contains all of
2525                         // the GC simulator tests.
2526                         if (scenario == 'longgc') {
2527                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2528                         }
2529                         else if (scenario == 'gcsimulator') {
2530                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2531                         }
2532                     }
2533                     else if (isGcReliabilityFramework(scenario)) {
2534                         testOpts += ' --build-overlay-only'
2535                     }
2536                     else if (scenario == 'standalone_gc') {
2537                         if (osGroup == 'OSX') {
2538                             testOpts += ' --gcname=libclrgc.dylib'
2539                         }
2540                         else if (osGroup == 'Linux') {
2541                             testOpts += ' --gcname=libclrgc.so'
2542                         }
2543                         else {
2544                             println("Unexpected OS group: ${osGroup} for os ${os}")
2545                             assert false
2546                         }
2547                     }
2548
2549                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2550
2551                     def folder = getJobFolder(scenario)
2552                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2553                         // Add parameters for the inputs
2554
2555                         if (windowsArmJob == true) {
2556                             parameters {
2557                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2558                             }
2559                         }
2560                         else {
2561                             parameters {
2562                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2563                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2564                             }
2565                         }
2566
2567                         steps {
2568                             // Set up the copies
2569
2570                             // Coreclr build containing the tests and mscorlib
2571                             // pri1 jobs still need to copy windows_nt built tests
2572                             if (windowsArmJob != true) {
2573                                 copyArtifacts(inputWindowsTestsBuildName) {
2574                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2575                                     buildSelector {
2576                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2577                                     }
2578                                 }
2579                             }
2580
2581                             // Coreclr build we are trying to test
2582                             //
2583                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2584
2585                             copyArtifacts(inputCoreCLRBuildName) {
2586                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2587                                 buildSelector {
2588                                     buildNumber('${CORECLR_BUILD}')
2589                                 }
2590                             }
2591
2592                             // Windows CoreCLR Arm(64) will restore corefx
2593                             // packages correctly.
2594                             //
2595                             // In addition, test steps are entirely different
2596                             // because we do not have a unified runner
2597                             if (windowsArmJob != true) {
2598                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2599
2600                                 // HACK -- Arm64 does not have corefx jobs yet.
2601                                 // Clone corefx and build the native packages overwriting the x64 packages.
2602                                 if (architecture == 'arm64') {
2603                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2604                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2605                                 }
2606                                 else if (architecture == 'x86') {
2607                                     shell("mkdir ./bin/CoreFxNative")
2608
2609                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2610                                         includePatterns('bin/build.tar.gz')
2611                                         targetDirectory('bin/CoreFxNative')
2612                                         buildSelector {
2613                                             latestSuccessful(true)
2614                                         }
2615                                     }
2616
2617                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2618                                 }
2619
2620                                 // Unzip the tests first.  Exit with 0
2621                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2622                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2623
2624                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2625
2626                                 // Execute the tests
2627                                 def runDocker = isNeedDocker(architecture, os, false)
2628                                 def dockerPrefix = ""
2629                                 def dockerCmd = ""
2630                                 if (runDocker) {
2631                                     def dockerImage = getDockerImageName(architecture, os, false)
2632                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2633                                     dockerCmd = dockerPrefix + "${dockerImage} "
2634                                 }
2635
2636                                 // If we are running a stress mode, we'll set those variables first
2637                                 def testEnvOpt = ""
2638                                 if (isJitStressScenario(scenario)) {
2639                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2640                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2641                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2642                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2643                                     shell("${envScriptCmds}")
2644                                     testEnvOpt = "--test-env=" + scriptFileName
2645                                 }
2646
2647                                 if (isGCStressRelatedTesting(scenario)) {
2648                                     shell('./init-tools.sh')
2649                                 }
2650
2651                                 shell("""${dockerCmd}./tests/runtest.sh \\
2652                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2653                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2654                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2655                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2656
2657                                 if (isGcReliabilityFramework(scenario)) {
2658                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2659                                     if (serverGCString != '') {
2660                                         if (runDocker) {
2661                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2662                                         }
2663                                         else {
2664                                             shell("export COMPlus_gcServer=1")
2665                                         }
2666                                     }
2667
2668                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2669                                 }
2670                             }
2671
2672                             else { // windowsArmJob == true
2673                                 // Unzip tests.
2674                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2675
2676                                 // Build the build commands
2677                                 def buildCommands = ""
2678
2679                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2680                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2681                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2682
2683                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2684                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2685
2686                                 // For all jobs
2687                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2688
2689                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2690                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2691
2692                                 // ARM legacy backend; this is an altjit.
2693                                 if (architecture == "armlb") {
2694                                     addEnvVariable("COMPlus_AltJit", "*")
2695                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2696                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2697                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2698                                 }
2699
2700                                 // If we are running a stress mode, we'll set those variables as well
2701                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2702                                     def stressValues = null
2703                                     if (isJitStressScenario(scenario)) {
2704                                         stressValues = Constants.jitStressModeScenarios[scenario]
2705                                     }
2706                                     else {
2707                                         stressValues = Constants.r2rStressScenarios[scenario]
2708                                     }
2709
2710                                     stressValues.each { key, value ->
2711                                         addEnvVariable(key, value)
2712                                     }
2713                                 }
2714
2715                                 if (isR2RScenario(scenario)) {
2716                                     // Crossgen the framework assemblies.
2717                                     buildCommands += """
2718 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2719 @goto skip_PrecompileAssembly
2720
2721 :PrecompileAssembly
2722 @REM Skip mscorlib since it is already precompiled.
2723 @if /I "%3" == "mscorlib.dll" exit /b 0
2724 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2725
2726 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2727 @if "%errorlevel%" == "-2146230517" (
2728     echo %2 is not a managed assembly.
2729 ) else if "%errorlevel%" == "-2146234344" (
2730     echo %2 is not a managed assembly.
2731 ) else if %errorlevel% neq 0 (
2732     echo Unable to precompile %2
2733 ) else (
2734     echo Precompiled %2
2735 )
2736 @exit /b 0
2737
2738 :skip_PrecompileAssembly
2739 """
2740
2741                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2742                                     // crossgen on tests before running them.
2743                                     addEnvVariable("RunCrossGen", "true")
2744                                 }
2745
2746                                 // Create the smarty command
2747                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2748                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2749                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2750
2751                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2752
2753                                 if (architecture == "armlb") {
2754                                     addExclude("LEGACYJIT_FAIL")
2755                                 }
2756
2757                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2758                                     def failTag = "JITSTRESS_FAIL"
2759                                     def excludeTag = "JITSTRESS_EXCLUDE"
2760
2761                                     if (scenario.contains('gc')) {
2762                                         failTag = "GCSTRESS_FAIL"
2763                                         excludeTag = "GCSTRESS_EXCLUDE"
2764                                     }
2765
2766                                     addArchSpecificExclude(architecture, failTag)
2767                                     addArchSpecificExclude(architecture, excludeTag)
2768                                 }
2769                                 else {
2770                                     addExclude("pri1")
2771                                 }
2772
2773                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2774                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2775                                 addExclude("LONG_RUNNING")
2776
2777                                 smartyCommand += "/lstFile Tests.lst"
2778
2779                                 def testListArch = [
2780                                     'arm64': 'arm64',
2781                                     'arm': 'arm',
2782                                     'armlb': 'arm'
2783                                 ]
2784
2785                                 def archLocation = testListArch[architecture]
2786
2787                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2788                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2789                                 addCommand("${smartyCommand}")
2790
2791                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2792                                 // However, we also need to remove all the variables that were set during this batch file, so we
2793                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2794                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2795                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2796                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2797                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2798                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2799
2800                                 // ZIP up the smarty output, no matter what the smarty result.
2801                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2802
2803                                 addCommand("echo %errorlevel%")
2804                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2805
2806                                 // Use the smarty errorlevel as the script errorlevel.
2807                                 addCommand("exit /b %__save_smarty_errorlevel%")
2808
2809                                 batchFile(buildCommands)
2810                             }
2811                         }
2812                     }
2813
2814                     addToViews(newJob, isPR, architecture, os)
2815
2816                     if (scenario == 'jitdiff') {
2817                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2818                     }
2819
2820                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2821                     if (os in ['Ubuntu']) {
2822                         SummaryBuilder summaries = new SummaryBuilder()
2823                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2824                         summaries.emit(newJob)
2825                     }
2826
2827                     def affinityOptions = null
2828
2829                     if (windowsArmJob == true) {
2830                         affinityOptions = [
2831                             "use_arm64_build_machine" : false
2832                         ]
2833                     }
2834
2835                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2836                         affinityOptions = [
2837                             "large_pages" : false
2838                         ]
2839                     }
2840
2841                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2842                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2843
2844                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2845
2846                     if (windowsArmJob != true) {
2847                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2848                     }
2849                     else {
2850                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2851
2852                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2853                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2854                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2855                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2856                     }
2857
2858                     // Create a build flow to join together the build and tests required to run this test.
2859                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2860                     // Linux CoreCLR test
2861                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2862                     def fullTestJobName = projectFolder + '/' + newJob.name
2863                     // Add a reference to the input jobs for report purposes
2864                     JobReport.Report.addReference(inputCoreCLRBuildName)
2865                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2866                     JobReport.Report.addReference(fullTestJobName)
2867                     def newFlowJob = null
2868
2869                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2870                         // Do not create the flow job for RHEL jobs.
2871                         return
2872                     }
2873
2874                     // For pri0 jobs we can build tests on unix
2875                     if (windowsArmJob) {
2876                         // For Windows arm jobs there is no reason to build a parallel test job.
2877                         // The product build supports building and archiving the tests.
2878
2879                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2880                         buildFlow("""
2881 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2882
2883 // And then build the test build
2884 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2885 """)
2886                         }
2887                     }
2888                     else {
2889                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2890                         buildFlow("""
2891 // Build the input jobs in parallel
2892 parallel (
2893 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2894 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2895 )
2896
2897 // And then build the test build
2898 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2899             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2900 """)
2901                         }
2902                     }
2903
2904                     addToViews(newFlowJob, isPR, architecture, os)
2905
2906                     // For the flow jobs set the machine affinity as x64 if an armarch.
2907                     def flowArch = architecture
2908
2909                     if (flowArch in validWindowsNTCrossArches) {
2910                         flowArch = 'x64'
2911                         affinityOptions = null
2912                     }
2913
2914                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2915                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2916                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2917                 } // configuration
2918             } // os
2919         } // architecture
2920     } // isPR
2921 } // scenario
2922
2923 JobReport.Report.generateJobReport(out)
2924
2925 // Make the call to generate the help job
2926 Utilities.createHelperJob(this, project, branch,
2927     "Welcome to the ${project} Repository",  // This is prepended to the help message
2928     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2929
2930 Utilities.addCROSSCheck(this, project, branch)