netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 // **************************************************************
 258 // Create some specific views
 259 //
 260 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 261 // views based on a single regular expression. These views will be generated by adding a
 262 // specific set of jobs to them.
 263 //
 264 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 265 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 266 // those are required, don't add them (which simplifies the view pages, as well).
 267 // **************************************************************
 268
 269 class Views {
 270     def static MergeJobView = null
 271     def static PeriodicJobView = null
 272     def static ArchitectureViews = [:]
 273     def static OSViews = [:]
 274 }
 275
 276 // MergeJobView: include all jobs that execute when a PR change is merged.
 277 Views.MergeJobView = listView('Merge') {
 278     recurse()
 279     columns {
 280         status()
 281         weather()
 282         name()
 283         lastSuccess()
 284         lastFailure()
 285         lastDuration()
 286         buildButton()
 287     }
 288 }
 289
 290 // PeriodicJobView: include all jobs that execute on a schedule
 291 Views.PeriodicJobView = listView('Periodic') {
 292     recurse()
 293     columns {
 294         status()
 295         weather()
 296         name()
 297         lastSuccess()
 298         lastFailure()
 299         lastDuration()
 300         buildButton()
 301     }
 302 }
 303
 304 // Create a view for non-PR jobs for each architecture.
 305 Constants.architectureList.each { architecture ->
 306     Views.ArchitectureViews[architecture] = listView(architecture) {
 307         recurse()
 308         columns {
 309             status()
 310             weather()
 311             name()
 312             lastSuccess()
 313             lastFailure()
 314             lastDuration()
 315             buildButton()
 316         }
 317     }
 318 }
 319
 320 // Create a view for non-PR jobs for each OS.
 321 Constants.osList.each { os ->
 322     // Don't create one for the special 'Windows_NT_BuildOnly'
 323     if (os == 'Windows_NT_BuildOnly') {
 324         return
 325     }
 326     Views.OSViews[os] = listView(os) {
 327         recurse()
 328         columns {
 329             status()
 330             weather()
 331             name()
 332             lastSuccess()
 333             lastFailure()
 334             lastDuration()
 335             buildButton()
 336         }
 337     }
 338 }
 339
 340 def static addToMergeView(def job) {
 341     Views.MergeJobView.with {
 342         jobs {
 343             name(job.name)
 344         }
 345     }
 346 }
 347
 348 def static addToPeriodicView(def job) {
 349     Views.PeriodicJobView.with {
 350         jobs {
 351             name(job.name)
 352         }
 353     }
 354 }
 355
 356 def static addToViews(def job, def isPR, def architecture, def os) {
 357     if (isPR) {
 358         // No views want PR jobs currently.
 359         return
 360     }
 361
 362     // Add to architecture view.
 363     Views.ArchitectureViews[architecture].with {
 364         jobs {
 365             name(job.name)
 366         }
 367     }
 368
 369     // Add to OS view.
 370     Views.OSViews[os].with {
 371         jobs {
 372             name(job.name)
 373         }
 374     }
 375 }
 376
 377 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 378     addToPeriodicView(job)
 379     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 380 }
 381
 382 def static addGithubPushTriggerHelper(def job) {
 383     addToMergeView(job)
 384     Utilities.addGithubPushTrigger(job)
 385 }
 386
 387
 388 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 389     assert os instanceof String
 390     assert architecture instanceof String
 391
 392     def armArches = ['arm', 'armlb', 'arm64']
 393     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 394
 395     if (!(architecture in armArches)) {
 396         assert options == null
 397         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 398
 399         return
 400     }
 401
 402     // This is an arm(64) job.
 403     //
 404     // There are several options.
 405     //
 406     // Windows_NT
 407     //
 408     // Arm32 (Build) -> latest-arm64
 409     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 410     // Arm32 (Test)  -> arm64-windows_nt
 411     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 412     //
 413     // Arm64 (Build) -> latest-arm64
 414     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 415     // Arm64 (Test)  -> arm64-windows_nt
 416     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 417     //
 418     // Ubuntu
 419     //
 420     // Arm32 (Build) -> arm-cross-latest
 421     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 422     // Arm32 (Test)  -> NYI Arch not supported
 423     //       |->
 424     //
 425     // Arm64 (Build) -> arm64-cross-latest
 426     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 427     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 428     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 429     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 430     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 431
 432     // This has to be a arm arch
 433     assert architecture in armArches
 434     if (os == "Windows_NT") {
 435         // Arm(64) Windows jobs share the same machines for now
 436         def isBuild = options['use_arm64_build_machine'] == true
 437
 438         if (isBuild == true) {
 439             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 440         } else {
 441             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 442         }
 443     } else {
 444         assert os != 'Windows_NT'
 445         assert os in supportedArmLinuxOs
 446
 447         if (architecture == 'arm' || architecture == 'armlb') {
 448             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 449         } else {
 450             // Arm64 Linux
 451             if (options['is_build_only'] == true) {
 452                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 453             } else {
 454                 // Arm64 Test Machines
 455                 if (options['large_pages'] == false) {
 456                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 457                 } else {
 458                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 459                 }
 460             }
 461         }
 462     }
 463 }
 464
 465 def static isGCStressRelatedTesting(def scenario) {
 466     // The 'r2r_gcstress15' scenario is a basic scenario.
 467     // Detect it and make it a GCStress related.
 468     if (scenario == 'r2r_gcstress15')
 469     {
 470         return true;
 471     }
 472
 473     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 474     def scenarioName = scenario.toLowerCase()
 475     def isGCStressTesting = false
 476     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 477         if (k in gcStressTestEnvVars) {
 478             isGCStressTesting = true;
 479         }
 480     }
 481     return isGCStressTesting
 482 }
 483
 484 def static isCoreFxScenario(def scenario) {
 485     def corefx_prefix = 'corefx_'
 486     if (scenario.length() < corefx_prefix.length()) {
 487         return false
 488     }
 489     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 490 }
 491
 492 def static isR2RBaselineScenario(def scenario) {
 493     return (scenario == 'r2r')
 494 }
 495
 496 def static isR2RStressScenario(def scenario) {
 497     return Constants.r2rStressScenarios.containsKey(scenario)
 498 }
 499
 500 def static isR2RScenario(def scenario) {
 501     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 502 }
 503
 504 def static isJitStressScenario(def scenario) {
 505     return Constants.jitStressModeScenarios.containsKey(scenario)
 506 }
 507
 508 def static isLongGc(def scenario) {
 509     return (scenario == 'longgc' || scenario == 'gcsimulator')
 510 }
 511
 512 def static isJitDiff(def scenario) {
 513     return (scenario == 'jitdiff')
 514 }
 515
 516 def static isGcReliabilityFramework(def scenario) {
 517     return (scenario == 'gc_reliability_framework')
 518 }
 519
 520 def static isArmWindowsScenario(def scenario) {
 521     return Constants.validArmWindowsScenarios.containsKey(scenario)
 522 }
 523
 524 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 525     // 2 hours (120 minutes) is the default timeout
 526     def timeout = 120
 527
 528     if (!(scenario == 'default' && isPR == true)) {
 529         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 530         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 531         timeout = 240
 532     }
 533
 534     if (!isBuildOnly) {
 535         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 536         if (isGCStressRelatedTesting(scenario)) {
 537             timeout = 4320
 538         }
 539         else if (isCoreFxScenario(scenario)) {
 540             timeout = 360
 541         }
 542         else if (isJitStressScenario(scenario)) {
 543             timeout = 240
 544         }
 545         else if (isR2RBaselineScenario(scenario)) {
 546             timeout = 240
 547         }
 548         else if (isLongGc(scenario)) {
 549             timeout = 1440
 550         }
 551         else if (isJitDiff(scenario)) {
 552             timeout = 240
 553         }
 554         else if (isGcReliabilityFramework(scenario)) {
 555             timeout = 1440
 556         }
 557         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 558             timeout = 240
 559         }
 560     }
 561
 562     if (configuration == 'Debug') {
 563         // Debug runs can be very slow. Add an hour.
 564         timeout += 60
 565     }
 566
 567     // If we've changed the timeout from the default, set it in the job.
 568
 569     if (timeout != 120) {
 570         Utilities.setJobTimeout(newJob, timeout)
 571     }
 572 }
 573
 574 def static getJobFolder(def scenario) {
 575     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 576         return 'jitstress'
 577     }
 578     if (scenario == 'illink') {
 579         return 'illink'
 580     }
 581     return ''
 582 }
 583
 584 def static getStressModeDisplayName(def scenario) {
 585     def displayStr = ''
 586     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 587         def prefixLength = 'COMPlus_'.length()
 588         if (k.length() >= prefixLength) {
 589             def modeName = k.substring(prefixLength, k.length())
 590             displayStr += ' ' + modeName + '=' + v
 591         }
 592     }
 593
 594     if (isCoreFxScenario(scenario)) {
 595         displayStr = ('CoreFx ' + displayStr).trim()
 596     }
 597
 598     return displayStr
 599 }
 600
 601 def static getR2RDisplayName(def scenario) {
 602     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 603     def displayStr = scenario
 604     def prefixLength = 'r2r_'.length()
 605     if (displayStr.length() >= prefixLength) {
 606         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 607     } else if (scenario == 'r2r') {
 608         displayStr = "R2R"
 609     }
 610     return displayStr
 611 }
 612
 613 //
 614 // Functions to create an environment script.
 615 //      envScriptCreate -- initialize the script (call first)
 616 //      envScriptFinalize -- finalize the script (call last)
 617 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 618 //      envScriptAppendExistingScript -- append an existing script to the generated script
 619 //
 620 // Each script returns a string of commands. Concatenate all the strings together before
 621 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 622 //
 623
 624 // Initialize the environment setting script.
 625 def static envScriptCreate(def os, def stepScriptLocation) {
 626     def stepScript = ''
 627     if (os == 'Windows_NT') {
 628         stepScript += "echo Creating TestEnv script\r\n"
 629         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 630
 631         // Create at least an empty script.
 632         stepScript += "echo. > ${stepScriptLocation}\r\n"
 633     }
 634     else {
 635         stepScript += "echo Creating environment setting script\n"
 636         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 637     }
 638
 639     return stepScript
 640 }
 641
 642 // Generates the string for setting stress mode variables.
 643 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 644     def stepScript = ''
 645     if (os == 'Windows_NT') {
 646         stressModeVars.each{ k, v ->
 647             // Write out what we are writing to the script file
 648             stepScript += "echo Setting ${k}=${v}\r\n"
 649             // Write out the set itself to the script file`
 650             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 651         }
 652     }
 653     else {
 654         stressModeVars.each{ k, v ->
 655             // Write out what we are writing to the script file
 656             stepScript += "echo Setting ${k}=${v}\n"
 657             // Write out the set itself to the script file`
 658             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 659         }
 660     }
 661
 662     return stepScript
 663 }
 664
 665 // Append an existing script to an environment script.
 666 // Returns string of commands to do this.
 667 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 668     assert (os == 'Windows_NT')
 669     def stepScript = ''
 670
 671     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 672     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 673
 674     return stepScript
 675 }
 676
 677 // Finalize an environment setting script.
 678 // Returns string of commands to do this.
 679 def static envScriptFinalize(def os, def stepScriptLocation) {
 680     def stepScript = ''
 681
 682     if (os == 'Windows_NT') {
 683         // Display the resulting script. This is useful when looking at the output log file.
 684         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 685         stepScript += "type ${stepScriptLocation}\r\n"
 686     }
 687     else {
 688         stepScript += "chmod +x ${stepScriptLocation}\n"
 689     }
 690
 691     return stepScript
 692 }
 693
 694 def static isNeedDocker(def architecture, def os, def isBuild) {
 695     if (isBuild) {
 696         if (architecture == 'x86' && os == 'Ubuntu') {
 697             return true
 698         }
 699         else if (architecture == 'arm') {
 700             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 701                 return true
 702             }
 703         }
 704     }
 705     else {
 706         if (architecture == 'x86' && os == 'Ubuntu') {
 707             return true
 708         }
 709     }
 710     return false
 711 }
 712
 713 def static getDockerImageName(def architecture, def os, def isBuild) {
 714     // We must change some docker private images to official later
 715     if (isBuild) {
 716         if (architecture == 'x86' && os == 'Ubuntu') {
 717             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 718         }
 719         else if (architecture == 'arm') {
 720             if (os == 'Ubuntu') {
 721                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 722             }
 723             else if (os == 'Ubuntu16.04') {
 724                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 725             }
 726             else if (os == 'Tizen') {
 727                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 728             }
 729         }
 730     }
 731     else {
 732         if (architecture == 'x86' && os == 'Ubuntu') {
 733             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 734         }
 735     }
 736     println("Unknown architecture to use docker: ${architecture} ${os}");
 737     assert false
 738 }
 739
 740 // Calculates the name of the build job based on some typical parameters.
 741 //
 742 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 743     // If the architecture is x64, do not add that info into the build name.
 744     // Need to change around some systems and other builds to pick up the right builds
 745     // to do that.
 746
 747     def suffix = scenario != 'default' ? "_${scenario}" : '';
 748     if (isBuildOnly) {
 749         suffix += '_bld'
 750     }
 751     def baseName = ''
 752     switch (architecture) {
 753         case 'x64':
 754             if (scenario == 'default') {
 755                 // For now we leave x64 off of the name for compatibility with other jobs
 756                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 757             }
 758             else if (scenario == 'formatting') {
 759                 // we don't care about the configuration for the formatting job. It runs all configs
 760                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 761             }
 762             else {
 763                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 764             }
 765             break
 766         case 'arm64':
 767             if (os.toLowerCase() == "windows_nt") {
 768                 // These are cross builds
 769                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 770             }
 771             else {
 772                 // Defaults to a small page size set of machines.
 773                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 774             }
 775             break
 776         case 'arm':
 777             // These are cross builds
 778             if (os == 'Tizen') {
 779                 // ABI: softfp
 780                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 781             }
 782             else {
 783                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 784             }
 785             break
 786         case 'armlb':
 787             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 788             break
 789         case 'x86':
 790         case 'x86_arm_altjit':
 791         case 'x64_arm64_altjit':
 792             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 793             break
 794         default:
 795             println("Unknown architecture: ${architecture}");
 796             assert false
 797             break
 798     }
 799
 800     return baseName + suffix
 801 }
 802
 803 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 804
 805     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 806     if (os == 'Windows_NT') {
 807         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 808             if (!(branch in Constants.WindowsArm64Branches)) {
 809                 return
 810             }
 811         }
 812     }
 813
 814     // Check scenario.
 815     switch (scenario) {
 816         case 'default':
 817             switch (architecture) {
 818                 case 'x64':
 819                 case 'x86':
 820                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 821                         addPeriodicTriggerHelper(job, '@daily')
 822                     }
 823                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 824                         addGithubPushTriggerHelper(job)
 825                     }
 826                     break
 827                 case 'arm':
 828                 case 'armlb':
 829                 case 'x86_arm_altjit':
 830                 case 'x64_arm64_altjit':
 831                     addGithubPushTriggerHelper(job)
 832                     break
 833                 case 'arm64':
 834                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 835                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 836                     break
 837                 default:
 838                     println("Unknown architecture: ${architecture}");
 839                     assert false
 840                     break
 841             }
 842             break
 843         case 'r2r':
 844             assert !(os in bidailyCrossList)
 845             // r2r gets a push trigger for checked/release
 846             if (configuration == 'Checked' || configuration == 'Release') {
 847                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 848                 if (architecture == 'x64' && os != 'OSX10.12') {
 849                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 850                     if (isFlowJob || os == 'Windows_NT') {
 851                         addGithubPushTriggerHelper(job)
 852                     }
 853                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 854                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 855                     if (isFlowJob) {
 856                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 857                     }
 858                 }
 859                 // For x86, only add per-commit jobs for Windows
 860                 else if (architecture == 'x86') {
 861                     if (os == 'Windows_NT') {
 862                         addGithubPushTriggerHelper(job)
 863                     }
 864                 }
 865                 // arm64 r2r jobs should only run daily.
 866                 else if (architecture == 'arm64') {
 867                     if (os == 'Windows_NT') {
 868                         addPeriodicTriggerHelper(job, '@daily')
 869                     }
 870                 }
 871             }
 872             break
 873         case 'r2r_jitstress1':
 874         case 'r2r_jitstress2':
 875         case 'r2r_jitstressregs1':
 876         case 'r2r_jitstressregs2':
 877         case 'r2r_jitstressregs3':
 878         case 'r2r_jitstressregs4':
 879         case 'r2r_jitstressregs8':
 880         case 'r2r_jitstressregs0x10':
 881         case 'r2r_jitstressregs0x80':
 882         case 'r2r_jitstressregs0x1000':
 883         case 'r2r_jitminopts':
 884         case 'r2r_jitforcerelocs':
 885         case 'r2r_gcstress15':
 886             assert !(os in bidailyCrossList)
 887
 888             // GCStress=C is currently not supported on OS X
 889             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 890                 break
 891             }
 892
 893             // GC Stress 15 r2r gets a push trigger for checked/release
 894             if (configuration == 'Checked' || configuration == 'Release') {
 895                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 896                 if (architecture == 'x64') {
 897                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 898                     if (isFlowJob || os == 'Windows_NT') {
 899                         // Add a weekly periodic trigger
 900                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 901                     }
 902                 }
 903                 // For x86, only add per-commit jobs for Windows
 904                 else if (architecture == 'x86') {
 905                     if (os == 'Windows_NT') {
 906                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 907                     }
 908                 }
 909             }
 910             break
 911         case 'longgc':
 912             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 913             assert configuration == 'Release'
 914             assert architecture == 'x64'
 915             addPeriodicTriggerHelper(job, '@daily')
 916             // TODO: Add once external email sending is available again
 917             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 918             break
 919         case 'gcsimulator':
 920             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 921             assert configuration == 'Release'
 922             assert architecture == 'x64'
 923             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 924             // TODO: Add once external email sending is available again
 925             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 926             break
 927         case 'standalone_gc':
 928             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 929             assert (configuration == 'Release' || configuration == 'Checked')
 930             // TODO: Add once external email sending is available again
 931             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 932             addPeriodicTriggerHelper(job, '@daily')
 933             break
 934         case 'gc_reliability_framework':
 935             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 936             assert (configuration == 'Release' || configuration == 'Checked')
 937             // Only triggered by phrase.
 938             break
 939         case 'ilrt':
 940             assert !(os in bidailyCrossList)
 941             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 942             if (architecture == 'x64' && configuration == 'Release') {
 943                 // We don't expect to see a job generated except in these scenarios
 944                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 945                 if (isFlowJob || os == 'Windows_NT') {
 946                     addPeriodicTriggerHelper(job, '@daily')
 947                 }
 948             }
 949             break
 950         case 'jitdiff':
 951             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 952             assert configuration == 'Checked'
 953             assert (architecture == 'x64' || architecture == 'x86')
 954             addGithubPushTriggerHelper(job)
 955             break
 956         case 'formatting':
 957             assert (os == 'Windows_NT' || os == "Ubuntu")
 958             assert architecture == 'x64'
 959             addGithubPushTriggerHelper(job)
 960             break
 961         case 'jitstressregs1':
 962         case 'jitstressregs2':
 963         case 'jitstressregs3':
 964         case 'jitstressregs4':
 965         case 'jitstressregs8':
 966         case 'jitstressregs0x10':
 967         case 'jitstressregs0x80':
 968         case 'jitstressregs0x1000':
 969         case 'minopts':
 970         case 'forcerelocs':
 971         case 'jitstress1':
 972         case 'jitstress2':
 973         case 'jitstress2_jitstressregs1':
 974         case 'jitstress2_jitstressregs2':
 975         case 'jitstress2_jitstressregs3':
 976         case 'jitstress2_jitstressregs4':
 977         case 'jitstress2_jitstressregs8':
 978         case 'jitstress2_jitstressregs0x10':
 979         case 'jitstress2_jitstressregs0x80':
 980         case 'jitstress2_jitstressregs0x1000':
 981         case 'tailcallstress':
 982         case 'jitsse2only':
 983         case 'jitnosimd':
 984         case 'corefx_baseline':
 985         case 'corefx_minopts':
 986         case 'corefx_jitstress1':
 987         case 'corefx_jitstress2':
 988         case 'corefx_jitstressregs1':
 989         case 'corefx_jitstressregs2':
 990         case 'corefx_jitstressregs3':
 991         case 'corefx_jitstressregs4':
 992         case 'corefx_jitstressregs8':
 993         case 'corefx_jitstressregs0x10':
 994         case 'corefx_jitstressregs0x80':
 995         case 'corefx_jitstressregs0x1000':
 996         case 'zapdisable':
 997             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 998                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 999                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1000                     if (os == 'Windows_NT') {
1001                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1002                         addPeriodicTriggerHelper(job, '@weekly')
1003                     }
1004                 }
1005                 else {
1006                     addPeriodicTriggerHelper(job, '@daily')
1007                 }
1008             }
1009             break
1010         case 'heapverify1':
1011         case 'gcstress0x3':
1012             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1013                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1014                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1015                     if (os == 'Windows_NT') {
1016                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1017                         addPeriodicTriggerHelper(job, '@weekly')
1018                     }
1019                     // TODO: Add once external email sending is available again
1020                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1021                 }
1022                 else {
1023                     addPeriodicTriggerHelper(job, '@weekly')
1024                 }
1025             }
1026             break
1027         case 'gcstress0xc':
1028         case 'gcstress0xc_zapdisable':
1029         case 'gcstress0xc_zapdisable_jitstress2':
1030         case 'gcstress0xc_zapdisable_heapverify1':
1031         case 'gcstress0xc_jitstress1':
1032         case 'gcstress0xc_jitstress2':
1033         case 'gcstress0xc_minopts_heapverify1':
1034             // GCStress=C is currently not supported on OS X
1035             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1036                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1037                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1038                     if (os == 'Windows_NT') {
1039                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1040                         addPeriodicTriggerHelper(job, '@weekly')
1041                     }
1042                     // TODO: Add once external email sending is available again
1043                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1044                 }
1045                 else {
1046                     addPeriodicTriggerHelper(job, '@weekly')
1047                 }
1048             }
1049             break
1050
1051         case 'illink':
1052             // Testing on other operating systems TBD
1053             assert (os == 'Windows_NT' || os == 'Ubuntu')
1054             if (architecture == 'x64' || architecture == 'x86') {
1055                 if (configuration == 'Checked') {
1056                     addPeriodicTriggerHelper(job, '@daily')
1057                 }
1058             }
1059             break
1060
1061         case 'tieredcompilation':
1062         case 'corefx_tieredcompilation':
1063             // No periodic jobs just yet, still testing
1064             break
1065
1066         default:
1067             println("Unknown scenario: ${scenario}");
1068             assert false
1069             break
1070     }
1071     return
1072 }
1073
1074 // **************************
1075 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1076 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1077 // builds will do a couple extra steps.
1078 // **************************
1079
1080 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1081 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1082 // it's just used for internal builds
1083 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1084 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1085     if (isWindowsBuildOnlyJob) {
1086         return
1087     }
1088
1089     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1090     // Non pull request builds.
1091     if (!isPR) {
1092         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1093         return
1094     }
1095
1096      def arm64Users = [
1097         'AndyAyersMS',
1098         'briansull',
1099         'BruceForstall',
1100         'CarolEidt',
1101         'cmckinsey',
1102         'erozenfeld',
1103         'janvorli',
1104         'jashook',
1105         'JosephTremoulet',
1106         'pgodeq',
1107         'pgavlin',
1108         'rartemev',
1109         'russellhadley',
1110         'RussKeldorph',
1111         'sandreenko',
1112         'sdmaclea',
1113         'swaroop-sridhar',
1114         'jkotas',
1115         'markwilkie',
1116         'weshaggard'
1117     ]
1118
1119     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1120     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1121     def osGroup = getOSGroup(os)
1122     switch (architecture) {
1123         case 'x64': // editor brace matching: {
1124             if (scenario == 'formatting') {
1125                 assert configuration == 'Checked'
1126                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1127                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1128                 }
1129                 break
1130             }
1131
1132             switch (os) {
1133                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1134                 case 'Debian8.4':
1135                 case 'RHEL7.2':
1136                     if (scenario == 'default') {
1137                         assert !isFlowJob
1138                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1139                     }
1140                     break
1141                 case 'Ubuntu16.04':
1142                     assert !isFlowJob
1143                     assert scenario == 'default'
1144                     // Distinguish with the other architectures (arm and x86)
1145                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1146                     break
1147                 case 'Fedora24':
1148                 case 'Ubuntu16.10':
1149                     assert !isFlowJob
1150                     assert scenario == 'default'
1151                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1152                     break
1153                 case 'Ubuntu':
1154                     if (scenario == 'illink') {
1155                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1156                         break
1157                     }
1158                     // fall through
1159                 case 'OSX10.12':
1160                     // Triggers on the non-flow jobs aren't necessary here
1161                     // Corefx testing uses non-flow jobs.
1162                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1163                         break
1164                     }
1165                     switch (scenario) {
1166                         case 'default':
1167                             // OSX uses checked for default PR tests
1168                             if (configuration == 'Checked') {
1169                                 // Default trigger
1170                                 assert !job.name.contains("centos")
1171                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1172                             }
1173                             break
1174                         case 'jitdiff':
1175                             if (configuration == 'Checked') {
1176                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1177                             }
1178                             break
1179                         case 'ilrt':
1180                             if (configuration == 'Release') {
1181                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1182                             }
1183                             break
1184                         case 'longgc':
1185                             if (configuration == 'Release') {
1186                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1187                             }
1188                             break
1189                         case 'gcsimulator':
1190                             if (configuration == 'Release') {
1191                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1192                             }
1193                             break
1194                         case 'standalone_gc':
1195                             if (configuration == 'Release' || configuration == 'Checked') {
1196                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1197                             }
1198                             break
1199                         case 'gc_reliability_framework':
1200                             if (configuration == 'Release' || configuration == 'Checked') {
1201                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1202                             }
1203                             break
1204                         default:
1205                             if (isJitStressScenario(scenario)) {
1206                                 def displayStr = getStressModeDisplayName(scenario)
1207                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1208                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1209                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1210                             }
1211                             else if (isR2RScenario(scenario)) {
1212                                 if (configuration == 'Release' || configuration == 'Checked') {
1213                                     def displayStr = getR2RDisplayName(scenario)
1214                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1215                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1216                                 }
1217                             }
1218                             else {
1219                                 println("Unknown scenario: ${scenario}");
1220                                 assert false
1221                             }
1222                             break
1223                     }
1224                     break
1225
1226                 case 'CentOS7.1':
1227                     switch (scenario) {
1228                         case 'default':
1229                             // CentOS uses checked for default PR tests while debug is build only
1230                             if (configuration == 'Debug') {
1231                                 // Default trigger
1232                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1233                             }
1234
1235                             // Make sure this is a flow job to get build and test.
1236                             if (configuration == 'Checked' && isFlowJob) {
1237                                 assert job.name.contains("flow")
1238                                 // Default trigger
1239                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1240                             }
1241                             break
1242                         default:
1243                             if (isR2RScenario(scenario)) {
1244                                 if (configuration == 'Release' || configuration == 'Checked') {
1245                                     def displayStr = getR2RDisplayName(scenario)
1246                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1247                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1248                                 }
1249                             }
1250                             break
1251                     }
1252                     break
1253
1254                 case 'Windows_NT':
1255                     switch (scenario) {
1256                         case 'default':
1257                             // Default trigger
1258                             if (configuration == 'Checked') {
1259                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1260                             }
1261                             break
1262                         case 'jitdiff':
1263                             if (configuration == 'Checked') {
1264                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1265                             }
1266                             break
1267                         case 'ilrt':
1268                             if (configuration == 'Release') {
1269                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1270                             }
1271                             break
1272                         case 'longgc':
1273                             if (configuration == 'Release') {
1274                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1275                             }
1276                             break
1277                         case 'gcsimulator':
1278                             if (configuration == 'Release') {
1279                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1280                             }
1281                             break
1282                         case 'standalone_gc':
1283                             if (configuration == 'Release' || configuration == 'Checked') {
1284                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1285                             }
1286                             break
1287                         case 'gc_reliability_framework':
1288                             if (configuration == 'Release' || configuration == 'Checked') {
1289                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1290                             }
1291                             break
1292                         case 'illink':
1293                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1294                             break
1295                         default:
1296                             if (isJitStressScenario(scenario)) {
1297                                 def displayStr = getStressModeDisplayName(scenario)
1298                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1299                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1300                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1301                             }
1302                             else if (isR2RScenario(scenario)) {
1303                                 if (configuration == 'Release' || configuration == 'Checked') {
1304                                     def displayStr = getR2RDisplayName(scenario)
1305                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1306                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1307                                 }
1308                             }
1309                             else {
1310                                 println("Unknown scenario: ${scenario}");
1311                                 assert false
1312                             }
1313                             break
1314                     }
1315                     break
1316                 default:
1317                     println("Unknown os: ${os}");
1318                     assert false
1319                     break
1320             }
1321             break
1322         // editor brace matching: }
1323         case 'armlb':
1324         case 'arm': // editor brace matching: {
1325             switch (os) {
1326                 case 'Ubuntu':
1327                 case 'Ubuntu16.04':
1328                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1329                         break
1330                     }
1331                     assert scenario == 'default'
1332                     job.with {
1333                         publishers {
1334                             azureVMAgentPostBuildAction {
1335                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1336                             }
1337                         }
1338                     }
1339                     if (configuration == 'Debug') {
1340                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1341                     }
1342                     else {
1343                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1344                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1345                     }
1346                     break
1347                 case 'Tizen':
1348                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1349                         break
1350                     }
1351                     architecture='armel'
1352                     job.with {
1353                         publishers {
1354                             azureVMAgentPostBuildAction {
1355                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1356                             }
1357                         }
1358                     }
1359                     if (configuration == 'Checked') {
1360                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1361                     }
1362                     else {
1363                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1364                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1365                     }
1366                     break
1367                 case 'Windows_NT':
1368                     // Triggers on the non-flow jobs aren't necessary here
1369                     if (!isFlowJob) {
1370                         break
1371                     }
1372
1373                     // Set up a private trigger
1374                     def contextString = "${os} ${architecture} Cross ${configuration}"
1375                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1376                     if (scenario == 'default') {
1377                         contextString += " Innerloop"
1378                         triggerString += "\\W+Innerloop"
1379                     }
1380                     else {
1381                         contextString += " ${scenario}"
1382                         triggerString += "\\W+${scenario}"
1383                     }
1384
1385                     if (configuration == 'Debug') {
1386                         contextString += " Build"
1387                         triggerString += "\\W+Build"
1388                     } else {
1389                         contextString += " Build and Test"
1390                         triggerString += "\\W+Build and Test"
1391                     }
1392
1393                     triggerString += ".*"
1394
1395                     switch (scenario) {
1396                         case 'default':
1397                             // Only Checked is a default trigger.
1398                             if (configuration == 'Checked')
1399                             {
1400                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1401                             }
1402                             else
1403                             {
1404                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1405                             }
1406                             break
1407                         default:
1408                             // Stress jobs will use this code path.
1409                             if (isArmWindowsScenario(scenario)) {
1410                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1411                             }
1412                             break
1413                     }
1414                     break
1415                 default:
1416                     println("NYI os: ${os}");
1417                     assert false
1418                     break
1419             }
1420             break
1421         // editor brace matching: }
1422         case 'arm64': // editor brace matching: {
1423             // Set up a private trigger
1424             def contextString = "${os} ${architecture} Cross ${configuration}"
1425             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1426             if (scenario == 'default') {
1427                 contextString += " Innerloop"
1428                 triggerString += "\\W+Innerloop"
1429             }
1430             else {
1431                 contextString += " ${scenario}"
1432                 triggerString += "\\W+${scenario}"
1433             }
1434
1435             if (configuration == 'Debug') {
1436                 contextString += " Build"
1437                 triggerString += "\\W+Build"
1438             } else {
1439                 contextString += " Build and Test"
1440                 triggerString += "\\W+Build and Test"
1441             }
1442
1443             triggerString += ".*"
1444
1445             switch (os) {
1446                 case 'Ubuntu':
1447                 case 'Ubuntu16.04':
1448                     switch (scenario) {
1449                         case 'default':
1450                             if (configuration == 'Debug' && !isFlowJob) {
1451                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1452                             }
1453                             else {
1454                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1455                             }
1456
1457                             break
1458                         default:
1459                             if (isR2RScenario(scenario)) {
1460                                 if (configuration == 'Checked' || configuration == 'Release') {
1461                                     def displayStr = getR2RDisplayName(scenario)
1462                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1463                                 }
1464                             }
1465                             break
1466                     }
1467                     break
1468                 case 'Windows_NT':
1469                     // Triggers on the non-flow jobs aren't necessary here
1470                     if (!isFlowJob) {
1471                         break
1472                     }
1473
1474                     assert isArmWindowsScenario(scenario)
1475                     switch (scenario) {
1476                         case 'default':
1477                             if (configuration == 'Checked') {
1478                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1479                             }
1480                             else {
1481                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1482                             }
1483                             break
1484                         default:
1485                             // Stress jobs will use this code path.
1486                             if (isArmWindowsScenario(scenario)) {
1487                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1488                             }
1489                             break
1490                     }
1491                     break
1492                 default:
1493                     println("NYI os: ${os}");
1494                     assert false
1495                     break
1496             }
1497             break
1498         // editor brace matching: }
1499         case 'x86': // editor brace matching: {
1500             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1501             if (os == 'Ubuntu') {
1502                 // Triggers on the non-flow jobs aren't necessary here
1503                 if (!isFlowJob) {
1504                     break
1505                 }
1506                 // on-demand only for ubuntu x86
1507                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1508                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1509                 break
1510             }
1511             switch (scenario) {
1512                 case 'default':
1513                     if (configuration == 'Checked') {
1514                         assert !job.name.contains("centos")
1515                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1516                     }
1517                     else {
1518                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1519                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1520                     }
1521                     break
1522                 case 'ilrt':
1523                     if (configuration == 'Release') {
1524                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1525                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1526                     }
1527                     break
1528                 case 'longgc':
1529                     if (configuration == 'Release') {
1530                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1531                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1532                     }
1533                     break
1534                 case 'gcsimulator':
1535                     if (configuration == 'Release') {
1536                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1537                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1538                     }
1539                     break
1540                 case 'standalone_gc':
1541                     if (configuration == 'Release' || configuration == 'Checked') {
1542                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1543                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1544                     }
1545                     break
1546                 case 'illink':
1547                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1548                     break
1549                 default:
1550                     if (isJitStressScenario(scenario)) {
1551                         def displayStr = getStressModeDisplayName(scenario)
1552                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1553                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1554                     }
1555                     else if (isR2RScenario(scenario)) {
1556                         if (configuration == 'Release' || configuration == 'Checked') {
1557                             def displayStr = getR2RDisplayName(scenario)
1558                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1559                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1560                         }
1561                     }
1562                     else {
1563                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1564                         assert false
1565                     }
1566                     break
1567             }
1568             break
1569          // editor brace matching: }
1570         case 'x64_arm64_altjit':
1571         case 'x86_arm_altjit': // editor brace matching: {
1572             assert (os == 'Windows_NT')
1573             switch (scenario) {
1574                 case 'default':
1575                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1576                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1577                     break
1578                 default:
1579                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1580                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1581                     break
1582             }
1583             break
1584         // editor brace matching: }
1585         default:
1586             println("Unknown architecture: ${architecture}");
1587             assert false
1588             break
1589     }
1590 }
1591
1592 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1593     def buildCommands = [];
1594     def osGroup = getOSGroup(os)
1595     def lowerConfiguration = configuration.toLowerCase()
1596
1597     def priority = '1'
1598     if (scenario == 'default' && isPR == true) {
1599         priority = '0'
1600     }
1601
1602     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1603
1604     def enableCorefxTesting = isCoreFxScenario(scenario)
1605
1606     // Calculate the build steps, archival, and xunit results
1607     switch (os) {
1608         case 'Windows_NT': // editor brace matching: {
1609             switch (architecture) {
1610                 case 'x64':
1611                 case 'x86':
1612                 case 'x86_arm_altjit':
1613                 case 'x64_arm64_altjit':
1614                     def arch = architecture
1615                     def buildOpts = ''
1616                     if (architecture == 'x86_arm_altjit') {
1617                         arch = 'x86'
1618                     }
1619                     else if (architecture == 'x64_arm64_altjit') {
1620                         arch = 'x64'
1621                     }
1622
1623                     if (scenario == 'formatting') {
1624                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1625                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1626                         break
1627                     }
1628
1629                     if (scenario == 'illink') {
1630                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1631                     }
1632
1633                     // If it is a release build for windows, ensure PGO is used, else fail the build
1634                     if ((lowerConfiguration == 'release') &&
1635                         (scenario in Constants.basicScenarios) &&
1636                         (architecture != 'x86_arm_altjit') &&
1637                         (architecture != 'x64_arm64_altjit')) {
1638
1639                         buildOpts += ' -enforcepgo'
1640                     }
1641
1642                     if (enableCorefxTesting) {
1643                         buildOpts += ' skiptests';
1644                     } else {
1645                         buildOpts += " -priority=${priority}"
1646                     }
1647
1648                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1649                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1650                     // 35 characters long.
1651
1652                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1653
1654                     if (!isBuildOnly) {
1655                         def runtestArguments = ''
1656                         def testOpts = 'collectdumps'
1657
1658                         if (isR2RScenario(scenario)) {
1659
1660                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1661                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1662                             // to cause the tests to be crossgen'ed.
1663
1664                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1665                                 testOpts += ' crossgenaltjit protononjit.dll'
1666                             } else {
1667                                 testOpts += ' crossgen'
1668                             }
1669
1670                             testOpts += ' runcrossgentests'
1671                         }
1672                         else if (scenario == 'jitdiff') {
1673                             testOpts += ' jitdisasm crossgen'
1674                         }
1675                         else if (scenario == 'ilrt') {
1676                             testOpts += ' ilasmroundtrip'
1677                         }
1678                         else if (isLongGc(scenario)) {
1679                             testOpts += " ${scenario} sequential"
1680                         }
1681                         else if (scenario == 'standalone_gc') {
1682                             testOpts += ' gcname clrgc.dll'
1683                         }
1684                         else if (scenario == 'illink') {
1685                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1686                         }
1687
1688                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1689                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1690                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1691                         // which is read by the xunit harness.)
1692                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1693                         {
1694                             def timeout = 1800000
1695                             testOpts += " timeout ${timeout}"
1696                         }
1697
1698                         // If we are running a stress mode, we should write out the set of key
1699                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1700
1701                         def envScriptPath = ''
1702                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1703                             def buildCommandsStr = ''
1704                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1705                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1706
1707                             if (isJitStressScenario(scenario)) {
1708                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1709                             }
1710                             else if (isR2RStressScenario(scenario)) {
1711                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1712                             }
1713
1714                             if (architecture == 'x86_arm_altjit') {
1715                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1716                             }
1717                             else if (architecture == 'x64_arm64_altjit') {
1718                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1719                             }
1720
1721                             envScriptFinalize(os, envScriptPath)
1722
1723                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1724                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1725                             // into a single script.
1726                             buildCommands += buildCommandsStr
1727                         }
1728                         else if (architecture == 'x86_arm_altjit') {
1729                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1730                         }
1731                         else if (architecture == 'x64_arm64_altjit') {
1732                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1733                         }
1734                         if (envScriptPath != '') {
1735                             testOpts += " TestEnv ${envScriptPath}"
1736                         }
1737
1738                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1739
1740                         if (enableCorefxTesting) {
1741                             def workspaceRelativeFxRoot = "_/fx"
1742                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1743
1744                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1745
1746                             // Archive and process (only) the test results
1747                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1748                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1749
1750                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1751                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1752                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1753                         }
1754                         else if (isGcReliabilityFramework(scenario)) {
1755                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1756                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1757                         }
1758                         else {
1759                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1760                         }
1761                     }
1762
1763                     if (!enableCorefxTesting) {
1764                         // Run the rest of the build
1765                         // Build the mscorlib for the other OS's
1766                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1767                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1768
1769                         if (arch == "x64") {
1770                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1771                         }
1772
1773                         // Zip up the tests directory so that we don't use so much space/time copying
1774                         // 10s of thousands of files around.
1775                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1776
1777                         if (!isJitStressScenario(scenario)) {
1778                             // For windows, pull full test results and test drops for x86/x64.
1779                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1780                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1781                         }
1782
1783                         if (scenario == 'jitdiff') {
1784                             // retrive jit-dasm output for base commit, and run jit-diff
1785                             if (!isBuildOnly) {
1786                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1787                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1788                             }
1789                         }
1790
1791                         if (!isBuildOnly) {
1792                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1793                         }
1794                     }
1795                     break
1796                 case 'armlb':
1797                 case 'arm':
1798                     assert isArmWindowsScenario(scenario)
1799
1800                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1801                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1802
1803                     def buildArchitecture = 'arm'
1804
1805                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1806                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1807                     def armCrossgenOpt = ''
1808                     if (architecture == 'armlb') {
1809                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1810                     }
1811
1812                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1813                     priority = '1'
1814
1815                     // This is now a build only job. Do not run tests. Use the flow job.
1816                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1817
1818                     // Zip up the tests directory so that we don't use so much space/time copying
1819                     // 10s of thousands of files around.
1820                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1821
1822                     // Add archival.
1823                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1824                     break
1825                 case 'arm64':
1826                     assert isArmWindowsScenario(scenario)
1827
1828                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1829                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1830
1831                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1832                     priority = '1'
1833
1834                     // This is now a build only job. Do not run tests. Use the flow job.
1835                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1836
1837                     // Zip up the tests directory so that we don't use so much space/time copying
1838                     // 10s of thousands of files around.
1839                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1840
1841                     // Add archival.
1842                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1843                     break
1844                 default:
1845                     println("Unknown architecture: ${architecture}");
1846                     assert false
1847                     break
1848             }
1849             break
1850         // editor brace matching: }
1851         case 'Ubuntu':
1852         case 'Ubuntu16.04':
1853         case 'Ubuntu16.10':
1854         case 'Debian8.4':
1855         case 'OSX10.12':
1856         case 'CentOS7.1':
1857         case 'RHEL7.2':
1858         case 'Tizen':
1859         case 'Fedora24': // editor brace matching: {
1860             switch (architecture) {
1861                 case 'x64':
1862                 case 'x86':
1863                     if (architecture == 'x86' && os == 'Ubuntu') {
1864                         // build and PAL test
1865                         def dockerImage = getDockerImageName(architecture, os, true)
1866                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1867                         dockerImage = getDockerImageName(architecture, os, false)
1868                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1869                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1870                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1871                         break
1872                     }
1873
1874                     if (scenario == 'formatting') {
1875                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1876                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1877                         break
1878                     }
1879
1880                     if (scenario == 'illink') {
1881                         assert(os == 'Ubuntu')
1882                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1883                     }
1884
1885                     if (!enableCorefxTesting) {
1886                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1887                         // only on supported OS platforms.
1888                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1889                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1890
1891                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1892                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1893
1894                         // Basic archiving of the build
1895                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1896                         // And pal tests
1897                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1898                     }
1899                     else {
1900                         // Corefx stress testing
1901                         assert os == 'Ubuntu'
1902                         assert architecture == 'x64'
1903                         assert lowerConfiguration == 'checked'
1904                         assert isJitStressScenario(scenario)
1905
1906                         // Build coreclr
1907                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1908
1909                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1910
1911                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1912                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1913                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1914                         buildCommands += envScriptCmds
1915
1916                         // Build and text corefx
1917                         def workspaceRelativeFxRoot = "_/fx"
1918                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1919
1920                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1921
1922                         // Archive and process (only) the test results
1923                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1924                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1925                     }
1926                     break
1927                 case 'arm64':
1928                     if (!enableCorefxTesting) {
1929                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1930
1931                         // HACK -- Arm64 does not have corefx jobs yet.
1932                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1933                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1934                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1935                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1936
1937                         // Basic archiving of the build
1938                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1939                     }
1940                     break
1941                 case 'arm':
1942                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1943                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1944
1945                     // default values for Ubuntu
1946                     def arm_abi="arm"
1947                     def linuxCodeName="trusty"
1948                     if (os == 'Ubuntu16.04') {
1949                         linuxCodeName="xenial"
1950                     }
1951                     else if (os == 'Tizen') {
1952                         arm_abi="armel"
1953                         linuxCodeName="tizen"
1954                     }
1955
1956                     // Unzip the Windows test binaries first. Exit with 0
1957                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1958
1959                     // Unpack the corefx binaries
1960                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1961                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1962                     if (os != 'Tizen') {
1963                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1964                     }
1965                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1966                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1967                     if (os == 'Ubuntu') {
1968                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1969                     }
1970
1971                     // Call the ARM CI script to cross build and test using docker
1972                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1973                     --mode=docker \\
1974                     --${arm_abi} \\
1975                     --linuxCodeName=${linuxCodeName} \\
1976                     --buildConfig=${lowerConfiguration} \\
1977                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1978                     --coreFxBinDir=./bin/CoreFxBinDir \\
1979                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1980
1981                     // Basic archiving of the build, no pal tests
1982                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1983                     break
1984                 default:
1985                     println("Unknown architecture: ${architecture}");
1986                     assert false
1987                     break
1988             }
1989             break
1990         // editor brace matching: }
1991         default:
1992             println("Unknown os: ${os}");
1993             assert false
1994             break
1995     } // os
1996
1997     return buildCommands
1998 }
1999
2000 Constants.allScenarios.each { scenario ->
2001     [true, false].each { isPR ->
2002         Constants.architectureList.each { architecture ->
2003             Constants.configurationList.each { configuration ->
2004                 Constants.osList.each { os ->
2005                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2006                     // and reset the os to Windows_NT
2007                     def isBuildOnly = false
2008                     if (os == 'Windows_NT_BuildOnly') {
2009                         isBuildOnly = true
2010                         os = 'Windows_NT'
2011                     }
2012
2013                     // Tizen is only supported for arm architecture
2014                     if (os == 'Tizen' && architecture != 'arm') {
2015                         return
2016                     }
2017
2018                     // Skip totally unimplemented (in CI) configurations.
2019                     switch (architecture) {
2020                         case 'arm64':
2021                             if (os == 'Ubuntu16.04') {
2022                                 os = 'Ubuntu'
2023                             }
2024
2025                             // Windows and Ubuntu only
2026                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2027                                 return
2028                             }
2029                             break
2030                         case 'arm':
2031                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2032                                 return
2033                             }
2034                             break
2035                         case 'armlb':
2036                             if (os != 'Windows_NT') {
2037                                 return
2038                             }
2039                             break
2040                         case 'x86':
2041                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2042                                 return
2043                             }
2044                             break
2045                         case 'x86_arm_altjit':
2046                         case 'x64_arm64_altjit':
2047                             if (os != 'Windows_NT') {
2048                                 return
2049                             }
2050                             break
2051                         case 'x64':
2052                             // Everything implemented
2053                             break
2054                         default:
2055                             println("Unknown architecture: ${architecture}")
2056                             assert false
2057                             break
2058                     }
2059
2060                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2061                     // with checked builds)
2062                     if (isJitStressScenario(scenario)) {
2063                         if (configuration != 'Checked') {
2064                             return
2065                         }
2066
2067                         // Since these are just execution time differences,
2068                         // skip platforms that don't execute the tests here (Windows_NT only)
2069                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2070                         if (!isEnabledOS || isBuildOnly) {
2071                             return
2072                         }
2073
2074                         switch (architecture) {
2075                             case 'x64':
2076                             case 'x86':
2077                             case 'x86_arm_altjit':
2078                             case 'x64_arm64_altjit':
2079                                 // x86 ubuntu: default only
2080                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2081                                     return
2082                                 }
2083                                 // Windows: Everything implemented
2084                                 break
2085
2086                             default:
2087                                 // arm, arm64, armlb: stress is handled through flow jobs.
2088                                 return
2089                         }
2090                     }
2091                     else if (isR2RScenario(scenario)) {
2092                         if (os != 'Windows_NT') {
2093                             return
2094                         }
2095                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2096                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2097                             return
2098                         }
2099                     }
2100                     else {
2101                         // Skip scenarios
2102                         switch (scenario) {
2103                             case 'ilrt':
2104                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2105                                 // the default scenario build
2106                                 if (os != 'Windows_NT') {
2107                                     return
2108                                 }
2109                                 // Only x64 for now
2110                                 if (architecture != 'x64') {
2111                                     return
2112                                 }
2113                                 // Release only
2114                                 if (configuration != 'Release') {
2115                                     return
2116                                 }
2117                                 break
2118                             case 'jitdiff':
2119                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2120                                     return
2121                                 }
2122                                 if (architecture != 'x64') {
2123                                     return
2124                                 }
2125                                 if (configuration != 'Checked') {
2126                                     return
2127                                 }
2128                                 break
2129                             case 'longgc':
2130                             case 'gcsimulator':
2131                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2132                                     return
2133                                 }
2134                                 if (architecture != 'x64') {
2135                                     return
2136                                 }
2137                                 if (configuration != 'Release') {
2138                                     return
2139                                 }
2140                                 break
2141                             case 'gc_reliability_framework':
2142                             case 'standalone_gc':
2143                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2144                                     return
2145                                 }
2146
2147                                 if (architecture != 'x64') {
2148                                     return
2149                                 }
2150
2151                                 if (configuration != 'Release' && configuration != 'Checked') {
2152                                     return
2153                                 }
2154                                 break
2155                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2156                             case 'formatting':
2157                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2158                                     return
2159                                 }
2160                                 if (architecture != 'x64') {
2161                                     return
2162                                 }
2163                                 if (configuration != 'Checked') {
2164                                     return
2165                                 }
2166                                 if (isBuildOnly) {
2167                                     return
2168                                 }
2169                                 break
2170                             case 'illink':
2171                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2172                                     return
2173                                 }
2174                                 if (architecture != 'x64' && architecture != 'x86') {
2175                                     return
2176                                 }
2177                                 if (isBuildOnly) {
2178                                     return
2179                                 }
2180                                 break
2181                             case 'default':
2182                                 // Nothing skipped
2183                                 break
2184                             default:
2185                                 println("Unknown scenario: ${scenario}")
2186                                 assert false
2187                                 break
2188                         }
2189                     }
2190
2191                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2192                     // runtime behavior, not compile-time behavior, are not interesting.
2193                     switch (architecture) {
2194                         case 'x86_arm_altjit':
2195                         case 'x64_arm64_altjit':
2196                             if (isGCStressRelatedTesting(scenario)) {
2197                                 return
2198                             }
2199                             break
2200                         default:
2201                             break
2202                     }
2203
2204                     // Calculate names
2205                     def lowerConfiguration = configuration.toLowerCase()
2206                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2207                     def folderName = getJobFolder(scenario)
2208
2209                     // Create the new job
2210                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2211                     addToViews(newJob, isPR, architecture, os)
2212
2213                     def machineAffinityOptions = null
2214
2215                     if (os != 'Windows_NT') {
2216                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2217                     }
2218                     else {
2219                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2220                     }
2221
2222                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2223
2224                     // Add all the standard options
2225                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2226                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2227
2228                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2229                     def osGroup = getOSGroup(os)
2230
2231                     newJob.with {
2232                         steps {
2233                             if (os == 'Windows_NT') {
2234                                 buildCommands.each { buildCommand ->
2235                                     batchFile(buildCommand)
2236                                 }
2237                             }
2238                             else {
2239                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2240                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2241                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2242                                     // Define the Windows Tests and Corefx build job names
2243                                     def WindowsTestsName = projectFolder + '/' +
2244                                                            Utilities.getFullJobName(project,
2245                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2246                                                                                     false)
2247                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2248                                                        Utilities.getFolderName(branch)
2249
2250                                     // Copy the Windows test binaries and the Corefx build binaries
2251                                     copyArtifacts(WindowsTestsName) {
2252                                         includePatterns('bin/tests/tests.zip')
2253                                         buildSelector {
2254                                             latestSuccessful(true)
2255                                         }
2256                                     }
2257
2258                                     def arm_abi = 'arm'
2259                                     def corefx_os = 'linux'
2260                                     if (os == 'Tizen') {
2261                                         arm_abi = 'armel'
2262                                         corefx_os = 'tizen'
2263                                     }
2264
2265                                     // Let's use release CoreFX to test checked CoreCLR,
2266                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2267                                     def corefx_lowerConfiguration = lowerConfiguration
2268                                     if ( lowerConfiguration == 'checked' ) {
2269                                         corefx_lowerConfiguration='release'
2270                                     }
2271
2272                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2273                                         includePatterns('bin/build.tar.gz')
2274                                         buildSelector {
2275                                             latestSuccessful(true)
2276                                         }
2277                                     }
2278                                 }
2279
2280                                 buildCommands.each { buildCommand ->
2281                                     shell(buildCommand)
2282                                 }
2283                             }
2284                         }
2285                     } // newJob.with
2286
2287                 } // os
2288             } // configuration
2289         } // architecture
2290     } // isPR
2291 } // scenario
2292
2293
2294 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2295 Constants.allScenarios.each { scenario ->
2296     [true, false].each { isPR ->
2297         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2298             Constants.crossList.each { os ->
2299                 if (architecture == 'arm64') {
2300                     if (os != "Ubuntu" && os != "Windows_NT") {
2301                         return
2302                     }
2303                 } else if (architecture == 'arm' || architecture == 'armlb') {
2304                     if (os != 'Windows_NT') {
2305                         return
2306                     }
2307                 }
2308                 else if (architecture == 'x86') {
2309                     if (os != "Ubuntu") {
2310                         return
2311                     }
2312                 }
2313
2314                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2315
2316                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2317                     return
2318                 }
2319
2320                 Constants.configurationList.each { configuration ->
2321
2322                     // First, filter based on OS.
2323
2324                     if (os == 'Windows_NT') {
2325                         if (!isArmWindowsScenario(scenario)) {
2326                             return
2327                         }
2328                     }
2329                     else {
2330                         // Non-Windows
2331                         if (architecture == 'arm64') {
2332                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2333                                 return
2334                             }
2335                         }
2336                         else if (architecture == 'x86') {
2337                             // Linux/x86 only want default test
2338                             if (scenario != 'default') {
2339                                 return
2340                             }
2341                         }
2342                     }
2343
2344                     // For CentOS, we only want Checked/Release builds.
2345                     if (os == 'CentOS7.1') {
2346                         if (configuration != 'Checked' && configuration != 'Release') {
2347                             return
2348                         }
2349                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2350                             return
2351                         }
2352                     }
2353
2354                     // For RedHat and Debian, we only do Release builds.
2355                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2356                         if (configuration != 'Release') {
2357                             return
2358                         }
2359                         if (scenario != 'default') {
2360                             return
2361                         }
2362                     }
2363
2364                     // Next, filter based on scenario.
2365
2366                     if (isJitStressScenario(scenario)) {
2367                         if (configuration != 'Checked') {
2368                             return
2369                         }
2370                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2371                         if (isCoreFxScenario(scenario)) {
2372                             return
2373                         }
2374                     }
2375                     else if (isR2RBaselineScenario(scenario)) {
2376                         if (configuration != 'Checked' && configuration != 'Release') {
2377                             return
2378                         }
2379                     }
2380                     else if (isR2RStressScenario(scenario)) {
2381                         if (configuration != 'Checked') {
2382                             return
2383                         }
2384                     }
2385                     else {
2386                         // Skip scenarios
2387                         switch (scenario) {
2388                             case 'ilrt':
2389                             case 'longgc':
2390                             case 'gcsimulator':
2391                                 // Long GC tests take a long time on non-Release builds
2392                                 // ilrt is also Release only
2393                                 if (configuration != 'Release') {
2394                                     return
2395                                 }
2396                                 break
2397                             case 'jitdiff':
2398                                 if (configuration != 'Checked') {
2399                                     return;
2400                                 }
2401                                 break
2402                             case 'gc_reliability_framework':
2403                             case 'standalone_gc':
2404                                 if (configuration != 'Release' && configuration != 'Checked') {
2405                                     return
2406                                 }
2407                                 break
2408                             case 'formatting':
2409                                 return
2410                             case 'illink':
2411                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2412                                     return
2413                                 }
2414                                 break
2415                             case 'default':
2416                                 // Nothing skipped
2417                                 break
2418                             default:
2419                                 println("Unknown scenario: ${scenario}")
2420                                 assert false
2421                                 break
2422                         }
2423                     }
2424
2425                     // Done filtering. Now, create the jobs.
2426
2427                     def lowerConfiguration = configuration.toLowerCase()
2428                     def osGroup = getOSGroup(os)
2429                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2430
2431                     def inputCoreCLRBuildName = projectFolder + '/' +
2432                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2433
2434                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2435                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2436                     def testBuildScenario = scenario
2437                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2438                         testBuildScenario = 'default'
2439                     }
2440
2441                     def inputWindowsTestBuildArch = architecture
2442                     if (architecture == "arm64" && os != "Windows_NT") {
2443                         // Use the x64 test build for arm64 unix
2444                         inputWindowsTestBuildArch = "x64"
2445                     }
2446
2447                     def inputWindowsTestsBuildName = ""
2448
2449                     if (isJitStressScenario(scenario)) {
2450                         inputWindowsTestsBuildName = projectFolder + '/' +
2451                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2452                     } else {
2453                         inputWindowsTestsBuildName = projectFolder + '/' +
2454                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2455
2456                     }
2457
2458
2459                     // Enable Server GC for Ubuntu PR builds
2460                     def serverGCString = ''
2461                     if (os == 'Ubuntu' && isPR) {
2462                         serverGCString = '--useServerGC'
2463                     }
2464
2465                     def testOpts = ''
2466
2467                     if (isR2RScenario(scenario)) {
2468
2469                         testOpts += ' --crossgen --runcrossgentests'
2470
2471                         if (scenario == 'r2r_jitstress1') {
2472                             testOpts += ' --jitstress=1'
2473                         }
2474                         else if (scenario == 'r2r_jitstress2') {
2475                             testOpts += ' --jitstress=2'
2476                         }
2477                         else if (scenario == 'r2r_jitstressregs1') {
2478                             testOpts += ' --jitstressregs=1'
2479                         }
2480                         else if (scenario == 'r2r_jitstressregs2') {
2481                             testOpts += ' --jitstressregs=2'
2482                         }
2483                         else if (scenario == 'r2r_jitstressregs3') {
2484                             testOpts += ' --jitstressregs=3'
2485                         }
2486                         else if (scenario == 'r2r_jitstressregs4') {
2487                             testOpts += ' --jitstressregs=4'
2488                         }
2489                         else if (scenario == 'r2r_jitstressregs8') {
2490                             testOpts += ' --jitstressregs=8'
2491                         }
2492                         else if (scenario == 'r2r_jitstressregs0x10') {
2493                             testOpts += ' --jitstressregs=0x10'
2494                         }
2495                         else if (scenario == 'r2r_jitstressregs0x80') {
2496                             testOpts += ' --jitstressregs=0x80'
2497                         }
2498                         else if (scenario == 'r2r_jitstressregs0x1000') {
2499                             testOpts += ' --jitstressregs=0x1000'
2500                         }
2501                         else if (scenario == 'r2r_jitminopts') {
2502                             testOpts += ' --jitminopts'
2503                         }
2504                         else if (scenario == 'r2r_jitforcerelocs') {
2505                             testOpts += ' --jitforcerelocs'
2506                         }
2507                         else if (scenario == 'r2r_gcstress15') {
2508                             testOpts += ' --gcstresslevel=0xF'
2509                         }
2510                     }
2511                     else if (scenario == 'jitdiff') {
2512                         testOpts += ' --jitdisasm --crossgen'
2513                     }
2514                     else if (scenario == 'illink') {
2515                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2516                     }
2517                     else if (isLongGc(scenario)) {
2518                         // Long GC tests behave very poorly when they are not
2519                         // the only test running (many of them allocate until OOM).
2520                         testOpts += ' --sequential'
2521
2522                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2523                         // for running long GC and GCSimulator tests, respectively. We don't use them
2524                         // here because using a playlist file produces much more readable output on the CI machines
2525                         // and reduces running time.
2526                         //
2527                         // The Long GC playlist contains all of the tests that are
2528                         // going to be run. The GCSimulator playlist contains all of
2529                         // the GC simulator tests.
2530                         if (scenario == 'longgc') {
2531                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2532                         }
2533                         else if (scenario == 'gcsimulator') {
2534                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2535                         }
2536                     }
2537                     else if (isGcReliabilityFramework(scenario)) {
2538                         testOpts += ' --build-overlay-only'
2539                     }
2540                     else if (scenario == 'standalone_gc') {
2541                         if (osGroup == 'OSX') {
2542                             testOpts += ' --gcname=libclrgc.dylib'
2543                         }
2544                         else if (osGroup == 'Linux') {
2545                             testOpts += ' --gcname=libclrgc.so'
2546                         }
2547                         else {
2548                             println("Unexpected OS group: ${osGroup} for os ${os}")
2549                             assert false
2550                         }
2551                     }
2552
2553                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2554
2555                     def folder = getJobFolder(scenario)
2556                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2557                         // Add parameters for the inputs
2558
2559                         if (windowsArmJob == true) {
2560                             parameters {
2561                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2562                             }
2563                         }
2564                         else {
2565                             parameters {
2566                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2567                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2568                             }
2569                         }
2570
2571                         steps {
2572                             // Set up the copies
2573
2574                             // Coreclr build containing the tests and mscorlib
2575                             // pri1 jobs still need to copy windows_nt built tests
2576                             if (windowsArmJob != true) {
2577                                 copyArtifacts(inputWindowsTestsBuildName) {
2578                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2579                                     buildSelector {
2580                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2581                                     }
2582                                 }
2583                             }
2584
2585                             // Coreclr build we are trying to test
2586                             //
2587                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2588
2589                             copyArtifacts(inputCoreCLRBuildName) {
2590                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2591                                 buildSelector {
2592                                     buildNumber('${CORECLR_BUILD}')
2593                                 }
2594                             }
2595
2596                             // Windows CoreCLR Arm(64) will restore corefx
2597                             // packages correctly.
2598                             //
2599                             // In addition, test steps are entirely different
2600                             // because we do not have a unified runner
2601                             if (windowsArmJob != true) {
2602                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2603
2604                                 // HACK -- Arm64 does not have corefx jobs yet.
2605                                 // Clone corefx and build the native packages overwriting the x64 packages.
2606                                 if (architecture == 'arm64') {
2607                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2608                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2609                                 }
2610                                 else if (architecture == 'x86') {
2611                                     shell("mkdir ./bin/CoreFxNative")
2612
2613                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2614                                         includePatterns('bin/build.tar.gz')
2615                                         targetDirectory('bin/CoreFxNative')
2616                                         buildSelector {
2617                                             latestSuccessful(true)
2618                                         }
2619                                     }
2620
2621                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2622                                 }
2623
2624                                 // Unzip the tests first.  Exit with 0
2625                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2626                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2627
2628                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2629
2630                                 // Execute the tests
2631                                 def runDocker = isNeedDocker(architecture, os, false)
2632                                 def dockerPrefix = ""
2633                                 def dockerCmd = ""
2634                                 if (runDocker) {
2635                                     def dockerImage = getDockerImageName(architecture, os, false)
2636                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2637                                     dockerCmd = dockerPrefix + "${dockerImage} "
2638                                 }
2639
2640                                 // If we are running a stress mode, we'll set those variables first
2641                                 def testEnvOpt = ""
2642                                 if (isJitStressScenario(scenario)) {
2643                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2644                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2645                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2646                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2647                                     shell("${envScriptCmds}")
2648                                     testEnvOpt = "--test-env=" + scriptFileName
2649                                 }
2650
2651                                 if (isGCStressRelatedTesting(scenario)) {
2652                                     shell('./init-tools.sh')
2653                                 }
2654
2655                                 shell("""${dockerCmd}./tests/runtest.sh \\
2656                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2657                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2658                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2659                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2660
2661                                 if (isGcReliabilityFramework(scenario)) {
2662                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2663                                     if (serverGCString != '') {
2664                                         if (runDocker) {
2665                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2666                                         }
2667                                         else {
2668                                             shell("export COMPlus_gcServer=1")
2669                                         }
2670                                     }
2671
2672                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2673                                 }
2674                             }
2675
2676                             else { // windowsArmJob == true
2677                                 // Unzip tests.
2678                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2679
2680                                 // Build the build commands
2681                                 def buildCommands = ""
2682
2683                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2684                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2685                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2686
2687                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2688                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2689
2690                                 // For all jobs
2691                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2692
2693                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2694                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2695
2696                                 // ARM legacy backend; this is an altjit.
2697                                 if (architecture == "armlb") {
2698                                     addEnvVariable("COMPlus_AltJit", "*")
2699                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2700                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2701                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2702                                 }
2703
2704                                 // If we are running a stress mode, we'll set those variables as well
2705                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2706                                     def stressValues = null
2707                                     if (isJitStressScenario(scenario)) {
2708                                         stressValues = Constants.jitStressModeScenarios[scenario]
2709                                     }
2710                                     else {
2711                                         stressValues = Constants.r2rStressScenarios[scenario]
2712                                     }
2713
2714                                     stressValues.each { key, value ->
2715                                         addEnvVariable(key, value)
2716                                     }
2717                                 }
2718
2719                                 if (isR2RScenario(scenario)) {
2720                                     // Crossgen the framework assemblies.
2721                                     buildCommands += """
2722 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2723 @goto skip_PrecompileAssembly
2724
2725 :PrecompileAssembly
2726 @REM Skip mscorlib since it is already precompiled.
2727 @if /I "%3" == "mscorlib.dll" exit /b 0
2728 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2729
2730 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2731 @if "%errorlevel%" == "-2146230517" (
2732     echo %2 is not a managed assembly.
2733 ) else if "%errorlevel%" == "-2146234344" (
2734     echo %2 is not a managed assembly.
2735 ) else if %errorlevel% neq 0 (
2736     echo Unable to precompile %2
2737 ) else (
2738     echo Precompiled %2
2739 )
2740 @exit /b 0
2741
2742 :skip_PrecompileAssembly
2743 """
2744
2745                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2746                                     // crossgen on tests before running them.
2747                                     addEnvVariable("RunCrossGen", "true")
2748                                 }
2749
2750                                 // Create the smarty command
2751                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2752                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2753                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2754
2755                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2756
2757                                 if (architecture == "armlb") {
2758                                     addExclude("LEGACYJIT_FAIL")
2759                                 }
2760
2761                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2762                                     def failTag = "JITSTRESS_FAIL"
2763                                     def excludeTag = "JITSTRESS_EXCLUDE"
2764
2765                                     if (scenario.contains('gc')) {
2766                                         failTag = "GCSTRESS_FAIL"
2767                                         excludeTag = "GCSTRESS_EXCLUDE"
2768                                     }
2769
2770                                     addArchSpecificExclude(architecture, failTag)
2771                                     addArchSpecificExclude(architecture, excludeTag)
2772                                 }
2773                                 else {
2774                                     addExclude("pri1")
2775                                 }
2776
2777                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2778                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2779                                 addExclude("LONG_RUNNING")
2780
2781                                 smartyCommand += "/lstFile Tests.lst"
2782
2783                                 def testListArch = [
2784                                     'arm64': 'arm64',
2785                                     'arm': 'arm',
2786                                     'armlb': 'arm'
2787                                 ]
2788
2789                                 def archLocation = testListArch[architecture]
2790
2791                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2792                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2793                                 addCommand("${smartyCommand}")
2794
2795                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2796                                 // However, we also need to remove all the variables that were set during this batch file, so we
2797                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2798                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2799                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2800                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2801                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2802                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2803
2804                                 // ZIP up the smarty output, no matter what the smarty result.
2805                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2806
2807                                 addCommand("echo %errorlevel%")
2808                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2809
2810                                 // Use the smarty errorlevel as the script errorlevel.
2811                                 addCommand("exit /b %__save_smarty_errorlevel%")
2812
2813                                 batchFile(buildCommands)
2814                             }
2815                         }
2816                     }
2817
2818                     addToViews(newJob, isPR, architecture, os)
2819
2820                     if (scenario == 'jitdiff') {
2821                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2822                     }
2823
2824                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2825                     if (os in ['Ubuntu']) {
2826                         SummaryBuilder summaries = new SummaryBuilder()
2827                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2828                         summaries.emit(newJob)
2829                     }
2830
2831                     def affinityOptions = null
2832
2833                     if (windowsArmJob == true) {
2834                         affinityOptions = [
2835                             "use_arm64_build_machine" : false
2836                         ]
2837                     }
2838
2839                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2840                         affinityOptions = [
2841                             "large_pages" : false
2842                         ]
2843                     }
2844
2845                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2846                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2847
2848                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2849
2850                     if (windowsArmJob != true) {
2851                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2852                     }
2853                     else {
2854                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2855
2856                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2857                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2858                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2859                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2860                     }
2861
2862                     // Create a build flow to join together the build and tests required to run this test.
2863                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2864                     // Linux CoreCLR test
2865                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2866                     def fullTestJobName = projectFolder + '/' + newJob.name
2867                     // Add a reference to the input jobs for report purposes
2868                     JobReport.Report.addReference(inputCoreCLRBuildName)
2869                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2870                     JobReport.Report.addReference(fullTestJobName)
2871                     def newFlowJob = null
2872
2873                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2874                         // Do not create the flow job for RHEL jobs.
2875                         return
2876                     }
2877
2878                     // For pri0 jobs we can build tests on unix
2879                     if (windowsArmJob) {
2880                         // For Windows arm jobs there is no reason to build a parallel test job.
2881                         // The product build supports building and archiving the tests.
2882
2883                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2884                         buildFlow("""
2885 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2886
2887 // And then build the test build
2888 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2889 """)
2890                         }
2891                     }
2892                     else {
2893                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2894                         buildFlow("""
2895 // Build the input jobs in parallel
2896 parallel (
2897 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2898 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2899 )
2900
2901 // And then build the test build
2902 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2903             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2904 """)
2905                         }
2906                     }
2907
2908                     addToViews(newFlowJob, isPR, architecture, os)
2909
2910                     // For the flow jobs set the machine affinity as x64 if an armarch.
2911                     def flowArch = architecture
2912
2913                     if (flowArch in validWindowsNTCrossArches) {
2914                         flowArch = 'x64'
2915                         affinityOptions = null
2916                     }
2917
2918                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2919                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2920                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2921                 } // configuration
2922             } // os
2923         } // architecture
2924     } // isPR
2925 } // scenario
2926
2927 JobReport.Report.generateJobReport(out)
2928
2929 // Make the call to generate the help job
2930 Utilities.createHelperJob(this, project, branch,
2931     "Welcome to the ${project} Repository",  // This is prepended to the help message
2932     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2933
2934 Utilities.addCROSSCheck(this, project, branch)