netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'default',
 139                'ilrt',
 140                'r2r',
 141                'longgc',
 142                'formatting',
 143                'gcsimulator',
 144                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 145                'standalone_gc',
 146                'gc_reliability_framework',
 147                'illink']
 148
 149     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 150
 151     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 152     // to Tests.lst file categories to exclude.
 153     //
 154     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 155     // and with the same values, with some commented out, for easier maintenance.
 156     //
 157     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 158     //
 159     def static validArmWindowsScenarios = [
 160                'default':                                [],
 161                // 'ilrt'
 162                'r2r':                                    ["R2R_FAIL"],
 163                // 'longgc'
 164                // 'formatting'
 165                // 'gcsimulator'
 166                // 'jitdiff'
 167                // 'standalone_gc'
 168                // 'gc_reliability_framework'
 169                // 'illink'
 170                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 184                'tieredcompilation':                      [],
 185                'forcerelocs':                            [],
 186                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 200                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 201                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 202                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 203                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 204                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 205                // 'jitsse2only'                          // Only relevant to xarch
 206                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 207                // 'corefx_baseline'
 208                // 'corefx_minopts'
 209                // 'corefx_tieredcompilation'
 210                // 'corefx_jitstress1'
 211                // 'corefx_jitstress2'
 212                // 'corefx_jitstressregs1'
 213                // 'corefx_jitstressregs2'
 214                // 'corefx_jitstressregs3'
 215                // 'corefx_jitstressregs4'
 216                // 'corefx_jitstressregs8'
 217                // 'corefx_jitstressregs0x10'
 218                // 'corefx_jitstressregs0x80'
 219                // 'corefx_jitstressregs0x1000'
 220                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 221                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 222                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 223                'heapverify1':                            [],
 224                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 225                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 226                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 227                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 230
 231                //
 232                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 233                //
 234
 235                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 236                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 254     ]
 255
 256     def static configurationList = ['Debug', 'Checked', 'Release']
 257
 258     // This is the set of architectures
 259     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 260 }
 261
 262 // **************************************************************
 263 // Create some specific views
 264 //
 265 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 266 // views based on a single regular expression. These views will be generated by adding a
 267 // specific set of jobs to them.
 268 //
 269 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 270 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 271 // those are required, don't add them (which simplifies the view pages, as well).
 272 // **************************************************************
 273
 274 class Views {
 275     def static MergeJobView = null
 276     def static PeriodicJobView = null
 277     def static ArchitectureViews = [:]
 278     def static OSViews = [:]
 279 }
 280
 281 // MergeJobView: include all jobs that execute when a PR change is merged.
 282 Views.MergeJobView = listView('Merge') {
 283     recurse()
 284     columns {
 285         status()
 286         weather()
 287         name()
 288         lastSuccess()
 289         lastFailure()
 290         lastDuration()
 291         buildButton()
 292     }
 293 }
 294
 295 // PeriodicJobView: include all jobs that execute on a schedule
 296 Views.PeriodicJobView = listView('Periodic') {
 297     recurse()
 298     columns {
 299         status()
 300         weather()
 301         name()
 302         lastSuccess()
 303         lastFailure()
 304         lastDuration()
 305         buildButton()
 306     }
 307 }
 308
 309 // Create a view for non-PR jobs for each architecture.
 310 Constants.architectureList.each { architecture ->
 311     Views.ArchitectureViews[architecture] = listView(architecture) {
 312         recurse()
 313         columns {
 314             status()
 315             weather()
 316             name()
 317             lastSuccess()
 318             lastFailure()
 319             lastDuration()
 320             buildButton()
 321         }
 322     }
 323 }
 324
 325 // Create a view for non-PR jobs for each OS.
 326 Constants.osList.each { os ->
 327     // Don't create one for the special 'Windows_NT_BuildOnly'
 328     if (os == 'Windows_NT_BuildOnly') {
 329         return
 330     }
 331     Views.OSViews[os] = listView(os) {
 332         recurse()
 333         columns {
 334             status()
 335             weather()
 336             name()
 337             lastSuccess()
 338             lastFailure()
 339             lastDuration()
 340             buildButton()
 341         }
 342     }
 343 }
 344
 345 def static addToMergeView(def job) {
 346     Views.MergeJobView.with {
 347         jobs {
 348             name(job.name)
 349         }
 350     }
 351 }
 352
 353 def static addToPeriodicView(def job) {
 354     Views.PeriodicJobView.with {
 355         jobs {
 356             name(job.name)
 357         }
 358     }
 359 }
 360
 361 def static addToViews(def job, def isPR, def architecture, def os) {
 362     if (isPR) {
 363         // No views want PR jobs currently.
 364         return
 365     }
 366
 367     // Add to architecture view.
 368     Views.ArchitectureViews[architecture].with {
 369         jobs {
 370             name(job.name)
 371         }
 372     }
 373
 374     // Add to OS view.
 375     Views.OSViews[os].with {
 376         jobs {
 377             name(job.name)
 378         }
 379     }
 380 }
 381
 382 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 383     addToPeriodicView(job)
 384     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 385 }
 386
 387 def static addGithubPushTriggerHelper(def job) {
 388     addToMergeView(job)
 389     Utilities.addGithubPushTrigger(job)
 390 }
 391
 392
 393 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 394     assert os instanceof String
 395     assert architecture instanceof String
 396
 397     def armArches = ['arm', 'armlb', 'arm64']
 398     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 399
 400     if (!(architecture in armArches)) {
 401         assert options == null
 402         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 403
 404         return
 405     }
 406
 407     // This is an arm(64) job.
 408     //
 409     // There are several options.
 410     //
 411     // Windows_NT
 412     //
 413     // Arm32 (Build) -> latest-arm64
 414     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 415     // Arm32 (Test)  -> arm64-windows_nt
 416     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 417     //
 418     // Arm64 (Build) -> latest-arm64
 419     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 420     // Arm64 (Test)  -> arm64-windows_nt
 421     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 422     //
 423     // Ubuntu
 424     //
 425     // Arm32 (Build) -> arm-cross-latest
 426     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 427     // Arm32 (Test)  -> NYI Arch not supported
 428     //       |->
 429     //
 430     // Arm64 (Build) -> arm64-cross-latest
 431     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 432     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 433     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 434     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 435     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 436
 437     // This has to be a arm arch
 438     assert architecture in armArches
 439     if (os == "Windows_NT") {
 440         // Arm(64) Windows jobs share the same machines for now
 441         def isBuild = options['use_arm64_build_machine'] == true
 442
 443         if (isBuild == true) {
 444             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 445         } else {
 446             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 447         }
 448     } else {
 449         assert os != 'Windows_NT'
 450         assert os in supportedArmLinuxOs
 451
 452         if (architecture == 'arm' || architecture == 'armlb') {
 453             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 454         } else {
 455             // Arm64 Linux
 456             if (options['is_build_only'] == true) {
 457                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 458             } else {
 459                 // Arm64 Test Machines
 460                 if (options['large_pages'] == false) {
 461                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 462                 } else {
 463                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 464                 }
 465             }
 466         }
 467     }
 468 }
 469
 470 def static isGCStressRelatedTesting(def scenario) {
 471     // The 'r2r_gcstress15' scenario is a basic scenario.
 472     // Detect it and make it a GCStress related.
 473     if (scenario == 'r2r_gcstress15')
 474     {
 475         return true;
 476     }
 477
 478     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 479     def scenarioName = scenario.toLowerCase()
 480     def isGCStressTesting = false
 481     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 482         if (k in gcStressTestEnvVars) {
 483             isGCStressTesting = true;
 484         }
 485     }
 486     return isGCStressTesting
 487 }
 488
 489 def static isCoreFxScenario(def scenario) {
 490     def corefx_prefix = 'corefx_'
 491     if (scenario.length() < corefx_prefix.length()) {
 492         return false
 493     }
 494     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 495 }
 496
 497 def static isR2RBaselineScenario(def scenario) {
 498     return (scenario == 'r2r')
 499 }
 500
 501 def static isR2RStressScenario(def scenario) {
 502     return Constants.r2rStressScenarios.containsKey(scenario)
 503 }
 504
 505 def static isR2RScenario(def scenario) {
 506     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 507 }
 508
 509 def static isJitStressScenario(def scenario) {
 510     return Constants.jitStressModeScenarios.containsKey(scenario)
 511 }
 512
 513 def static isLongGc(def scenario) {
 514     return (scenario == 'longgc' || scenario == 'gcsimulator')
 515 }
 516
 517 def static isJitDiff(def scenario) {
 518     return (scenario == 'jitdiff')
 519 }
 520
 521 def static isGcReliabilityFramework(def scenario) {
 522     return (scenario == 'gc_reliability_framework')
 523 }
 524
 525 def static isArmWindowsScenario(def scenario) {
 526     return Constants.validArmWindowsScenarios.containsKey(scenario)
 527 }
 528
 529 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 530     // 2 hours (120 minutes) is the default timeout
 531     def timeout = 120
 532
 533     if (!(scenario == 'default' && isPR == true)) {
 534         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 535         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 536         timeout = 240
 537     }
 538
 539     if (!isBuildOnly) {
 540         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 541         if (isGCStressRelatedTesting(scenario)) {
 542             timeout = 4320
 543         }
 544         else if (isCoreFxScenario(scenario)) {
 545             timeout = 360
 546         }
 547         else if (isJitStressScenario(scenario)) {
 548             timeout = 240
 549         }
 550         else if (isR2RBaselineScenario(scenario)) {
 551             timeout = 240
 552         }
 553         else if (isLongGc(scenario)) {
 554             timeout = 1440
 555         }
 556         else if (isJitDiff(scenario)) {
 557             timeout = 240
 558         }
 559         else if (isGcReliabilityFramework(scenario)) {
 560             timeout = 1440
 561         }
 562         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 563             timeout = 240
 564         }
 565     }
 566
 567     if (configuration == 'Debug') {
 568         // Debug runs can be very slow. Add an hour.
 569         timeout += 60
 570     }
 571
 572     // If we've changed the timeout from the default, set it in the job.
 573
 574     if (timeout != 120) {
 575         Utilities.setJobTimeout(newJob, timeout)
 576     }
 577 }
 578
 579 def static getJobFolder(def scenario) {
 580     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 581         return 'jitstress'
 582     }
 583     if (scenario == 'illink') {
 584         return 'illink'
 585     }
 586     return ''
 587 }
 588
 589 def static getStressModeDisplayName(def scenario) {
 590     def displayStr = ''
 591     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 592         def prefixLength = 'COMPlus_'.length()
 593         if (k.length() >= prefixLength) {
 594             def modeName = k.substring(prefixLength, k.length())
 595             displayStr += ' ' + modeName + '=' + v
 596         }
 597     }
 598
 599     if (isCoreFxScenario(scenario)) {
 600         displayStr = ('CoreFx ' + displayStr).trim()
 601     }
 602
 603     return displayStr
 604 }
 605
 606 def static getR2RDisplayName(def scenario) {
 607     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 608     def displayStr = scenario
 609     def prefixLength = 'r2r_'.length()
 610     if (displayStr.length() >= prefixLength) {
 611         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 612     } else if (scenario == 'r2r') {
 613         displayStr = "R2R"
 614     }
 615     return displayStr
 616 }
 617
 618 //
 619 // Functions to create an environment script.
 620 //      envScriptCreate -- initialize the script (call first)
 621 //      envScriptFinalize -- finalize the script (call last)
 622 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 623 //      envScriptAppendExistingScript -- append an existing script to the generated script
 624 //
 625 // Each script returns a string of commands. Concatenate all the strings together before
 626 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 627 //
 628
 629 // Initialize the environment setting script.
 630 def static envScriptCreate(def os, def stepScriptLocation) {
 631     def stepScript = ''
 632     if (os == 'Windows_NT') {
 633         stepScript += "echo Creating TestEnv script\r\n"
 634         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 635
 636         // Create at least an empty script.
 637         stepScript += "echo. > ${stepScriptLocation}\r\n"
 638     }
 639     else {
 640         stepScript += "echo Creating environment setting script\n"
 641         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 642     }
 643
 644     return stepScript
 645 }
 646
 647 // Generates the string for setting stress mode variables.
 648 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 649     def stepScript = ''
 650     if (os == 'Windows_NT') {
 651         stressModeVars.each{ k, v ->
 652             // Write out what we are writing to the script file
 653             stepScript += "echo Setting ${k}=${v}\r\n"
 654             // Write out the set itself to the script file`
 655             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 656         }
 657     }
 658     else {
 659         stressModeVars.each{ k, v ->
 660             // Write out what we are writing to the script file
 661             stepScript += "echo Setting ${k}=${v}\n"
 662             // Write out the set itself to the script file`
 663             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 664         }
 665     }
 666
 667     return stepScript
 668 }
 669
 670 // Append an existing script to an environment script.
 671 // Returns string of commands to do this.
 672 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 673     assert (os == 'Windows_NT')
 674     def stepScript = ''
 675
 676     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 677     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 678
 679     return stepScript
 680 }
 681
 682 // Finalize an environment setting script.
 683 // Returns string of commands to do this.
 684 def static envScriptFinalize(def os, def stepScriptLocation) {
 685     def stepScript = ''
 686
 687     if (os == 'Windows_NT') {
 688         // Display the resulting script. This is useful when looking at the output log file.
 689         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 690         stepScript += "type ${stepScriptLocation}\r\n"
 691     }
 692     else {
 693         stepScript += "chmod +x ${stepScriptLocation}\n"
 694     }
 695
 696     return stepScript
 697 }
 698
 699 def static isNeedDocker(def architecture, def os, def isBuild) {
 700     if (isBuild) {
 701         if (architecture == 'x86' && os == 'Ubuntu') {
 702             return true
 703         }
 704         else if (architecture == 'arm') {
 705             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 706                 return true
 707             }
 708         }
 709     }
 710     else {
 711         if (architecture == 'x86' && os == 'Ubuntu') {
 712             return true
 713         }
 714     }
 715     return false
 716 }
 717
 718 def static getDockerImageName(def architecture, def os, def isBuild) {
 719     // We must change some docker private images to official later
 720     if (isBuild) {
 721         if (architecture == 'x86' && os == 'Ubuntu') {
 722             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 723         }
 724         else if (architecture == 'arm') {
 725             if (os == 'Ubuntu') {
 726                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 727             }
 728             else if (os == 'Ubuntu16.04') {
 729                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 730             }
 731             else if (os == 'Tizen') {
 732                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 733             }
 734         }
 735     }
 736     else {
 737         if (architecture == 'x86' && os == 'Ubuntu') {
 738             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 739         }
 740     }
 741     println("Unknown architecture to use docker: ${architecture} ${os}");
 742     assert false
 743 }
 744
 745 // Calculates the name of the build job based on some typical parameters.
 746 //
 747 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 748     // If the architecture is x64, do not add that info into the build name.
 749     // Need to change around some systems and other builds to pick up the right builds
 750     // to do that.
 751
 752     def suffix = scenario != 'default' ? "_${scenario}" : '';
 753     if (isBuildOnly) {
 754         suffix += '_bld'
 755     }
 756     def baseName = ''
 757     switch (architecture) {
 758         case 'x64':
 759             if (scenario == 'default') {
 760                 // For now we leave x64 off of the name for compatibility with other jobs
 761                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 762             }
 763             else if (scenario == 'formatting') {
 764                 // we don't care about the configuration for the formatting job. It runs all configs
 765                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 766             }
 767             else {
 768                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 769             }
 770             break
 771         case 'arm64':
 772             if (os.toLowerCase() == "windows_nt") {
 773                 // These are cross builds
 774                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 775             }
 776             else {
 777                 // Defaults to a small page size set of machines.
 778                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 779             }
 780             break
 781         case 'arm':
 782             // These are cross builds
 783             if (os == 'Tizen') {
 784                 // ABI: softfp
 785                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 786             }
 787             else {
 788                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 789             }
 790             break
 791         case 'armlb':
 792             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 793             break
 794         case 'x86':
 795         case 'x86_arm_altjit':
 796         case 'x64_arm64_altjit':
 797             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 798             break
 799         default:
 800             println("Unknown architecture: ${architecture}");
 801             assert false
 802             break
 803     }
 804
 805     return baseName + suffix
 806 }
 807
 808 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 809
 810     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 811     if (os == 'Windows_NT') {
 812         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 813             if (!(branch in Constants.WindowsArm64Branches)) {
 814                 return
 815             }
 816         }
 817     }
 818
 819     // Check scenario.
 820     switch (scenario) {
 821         case 'default':
 822             switch (architecture) {
 823                 case 'x64':
 824                 case 'x86':
 825                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 826                         addPeriodicTriggerHelper(job, '@daily')
 827                     }
 828                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 829                         addGithubPushTriggerHelper(job)
 830                     }
 831                     break
 832                 case 'arm':
 833                 case 'armlb':
 834                 case 'x86_arm_altjit':
 835                 case 'x64_arm64_altjit':
 836                     addGithubPushTriggerHelper(job)
 837                     break
 838                 case 'arm64':
 839                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 840                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 841                     break
 842                 default:
 843                     println("Unknown architecture: ${architecture}");
 844                     assert false
 845                     break
 846             }
 847             break
 848         case 'r2r':
 849             assert !(os in bidailyCrossList)
 850             // r2r gets a push trigger for checked/release
 851             if (configuration == 'Checked' || configuration == 'Release') {
 852                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 853                 if (architecture == 'x64' && os != 'OSX10.12') {
 854                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 855                     if (isFlowJob || os == 'Windows_NT') {
 856                         addGithubPushTriggerHelper(job)
 857                     }
 858                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 859                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 860                     if (isFlowJob) {
 861                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 862                     }
 863                 }
 864                 // For x86, only add per-commit jobs for Windows
 865                 else if (architecture == 'x86') {
 866                     if (os == 'Windows_NT') {
 867                         addGithubPushTriggerHelper(job)
 868                     }
 869                 }
 870                 // arm64 r2r jobs should only run daily.
 871                 else if (architecture == 'arm64') {
 872                     if (os == 'Windows_NT') {
 873                         addPeriodicTriggerHelper(job, '@daily')
 874                     }
 875                 }
 876             }
 877             break
 878         case 'r2r_jitstress1':
 879         case 'r2r_jitstress2':
 880         case 'r2r_jitstressregs1':
 881         case 'r2r_jitstressregs2':
 882         case 'r2r_jitstressregs3':
 883         case 'r2r_jitstressregs4':
 884         case 'r2r_jitstressregs8':
 885         case 'r2r_jitstressregs0x10':
 886         case 'r2r_jitstressregs0x80':
 887         case 'r2r_jitstressregs0x1000':
 888         case 'r2r_jitminopts':
 889         case 'r2r_jitforcerelocs':
 890         case 'r2r_gcstress15':
 891             assert !(os in bidailyCrossList)
 892
 893             // GCStress=C is currently not supported on OS X
 894             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 895                 break
 896             }
 897
 898             // GC Stress 15 r2r gets a push trigger for checked/release
 899             if (configuration == 'Checked' || configuration == 'Release') {
 900                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 901                 if (architecture == 'x64') {
 902                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 903                     if (isFlowJob || os == 'Windows_NT') {
 904                         // Add a weekly periodic trigger
 905                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 906                     }
 907                 }
 908                 // For x86, only add per-commit jobs for Windows
 909                 else if (architecture == 'x86') {
 910                     if (os == 'Windows_NT') {
 911                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 912                     }
 913                 }
 914             }
 915             break
 916         case 'longgc':
 917             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 918             assert configuration == 'Release'
 919             assert architecture == 'x64'
 920             addPeriodicTriggerHelper(job, '@daily')
 921             // TODO: Add once external email sending is available again
 922             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 923             break
 924         case 'gcsimulator':
 925             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 926             assert configuration == 'Release'
 927             assert architecture == 'x64'
 928             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 929             // TODO: Add once external email sending is available again
 930             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 931             break
 932         case 'standalone_gc':
 933             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 934             assert (configuration == 'Release' || configuration == 'Checked')
 935             // TODO: Add once external email sending is available again
 936             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 937             addPeriodicTriggerHelper(job, '@daily')
 938             break
 939         case 'gc_reliability_framework':
 940             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 941             assert (configuration == 'Release' || configuration == 'Checked')
 942             // Only triggered by phrase.
 943             break
 944         case 'ilrt':
 945             assert !(os in bidailyCrossList)
 946             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 947             if (architecture == 'x64' && configuration == 'Release') {
 948                 // We don't expect to see a job generated except in these scenarios
 949                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 950                 if (isFlowJob || os == 'Windows_NT') {
 951                     addPeriodicTriggerHelper(job, '@daily')
 952                 }
 953             }
 954             break
 955         case 'jitdiff':
 956             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 957             assert configuration == 'Checked'
 958             assert (architecture == 'x64' || architecture == 'x86')
 959             addGithubPushTriggerHelper(job)
 960             break
 961         case 'formatting':
 962             assert (os == 'Windows_NT' || os == "Ubuntu")
 963             assert architecture == 'x64'
 964             addGithubPushTriggerHelper(job)
 965             break
 966         case 'jitstressregs1':
 967         case 'jitstressregs2':
 968         case 'jitstressregs3':
 969         case 'jitstressregs4':
 970         case 'jitstressregs8':
 971         case 'jitstressregs0x10':
 972         case 'jitstressregs0x80':
 973         case 'jitstressregs0x1000':
 974         case 'minopts':
 975         case 'forcerelocs':
 976         case 'jitstress1':
 977         case 'jitstress2':
 978         case 'jitstress2_jitstressregs1':
 979         case 'jitstress2_jitstressregs2':
 980         case 'jitstress2_jitstressregs3':
 981         case 'jitstress2_jitstressregs4':
 982         case 'jitstress2_jitstressregs8':
 983         case 'jitstress2_jitstressregs0x10':
 984         case 'jitstress2_jitstressregs0x80':
 985         case 'jitstress2_jitstressregs0x1000':
 986         case 'tailcallstress':
 987         case 'jitsse2only':
 988         case 'jitnosimd':
 989         case 'jitnox86hwintrinsic':
 990         case 'jitincompletehwintrinsic':
 991         case 'jitx86hwintrinsicnoavx':
 992         case 'jitx86hwintrinsicnoavx2':
 993         case 'jitx86hwintrinsicnosimd':
 994         case 'corefx_baseline':
 995         case 'corefx_minopts':
 996         case 'corefx_jitstress1':
 997         case 'corefx_jitstress2':
 998         case 'corefx_jitstressregs1':
 999         case 'corefx_jitstressregs2':
1000         case 'corefx_jitstressregs3':
1001         case 'corefx_jitstressregs4':
1002         case 'corefx_jitstressregs8':
1003         case 'corefx_jitstressregs0x10':
1004         case 'corefx_jitstressregs0x80':
1005         case 'corefx_jitstressregs0x1000':
1006         case 'zapdisable':
1007             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1008                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1009                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1010                     if (os == 'Windows_NT') {
1011                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1012                         addPeriodicTriggerHelper(job, '@weekly')
1013                     }
1014                 }
1015                 else {
1016                     addPeriodicTriggerHelper(job, '@daily')
1017                 }
1018             }
1019             break
1020         case 'heapverify1':
1021         case 'gcstress0x3':
1022             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1023                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1024                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1025                     if (os == 'Windows_NT') {
1026                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1027                         addPeriodicTriggerHelper(job, '@weekly')
1028                     }
1029                     // TODO: Add once external email sending is available again
1030                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1031                 }
1032                 else {
1033                     addPeriodicTriggerHelper(job, '@weekly')
1034                 }
1035             }
1036             break
1037         case 'gcstress0xc':
1038         case 'gcstress0xc_zapdisable':
1039         case 'gcstress0xc_zapdisable_jitstress2':
1040         case 'gcstress0xc_zapdisable_heapverify1':
1041         case 'gcstress0xc_jitstress1':
1042         case 'gcstress0xc_jitstress2':
1043         case 'gcstress0xc_minopts_heapverify1':
1044             // GCStress=C is currently not supported on OS X
1045             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1046                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1047                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1048                     if (os == 'Windows_NT') {
1049                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1050                         addPeriodicTriggerHelper(job, '@weekly')
1051                     }
1052                     // TODO: Add once external email sending is available again
1053                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1054                 }
1055                 else {
1056                     addPeriodicTriggerHelper(job, '@weekly')
1057                 }
1058             }
1059             break
1060
1061         case 'illink':
1062             // Testing on other operating systems TBD
1063             assert (os == 'Windows_NT' || os == 'Ubuntu')
1064             if (architecture == 'x64' || architecture == 'x86') {
1065                 if (configuration == 'Checked') {
1066                     addPeriodicTriggerHelper(job, '@daily')
1067                 }
1068             }
1069             break
1070
1071         case 'tieredcompilation':
1072         case 'corefx_tieredcompilation':
1073             // No periodic jobs just yet, still testing
1074             break
1075
1076         default:
1077             println("Unknown scenario: ${scenario}");
1078             assert false
1079             break
1080     }
1081     return
1082 }
1083
1084 // **************************
1085 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1086 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1087 // builds will do a couple extra steps.
1088 // **************************
1089
1090 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1091 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1092 // it's just used for internal builds
1093 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1094 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1095     if (isWindowsBuildOnlyJob) {
1096         return
1097     }
1098
1099     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1100     // Non pull request builds.
1101     if (!isPR) {
1102         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1103         return
1104     }
1105
1106      def arm64Users = [
1107         'AndyAyersMS',
1108         'briansull',
1109         'BruceForstall',
1110         'CarolEidt',
1111         'cmckinsey',
1112         'erozenfeld',
1113         'janvorli',
1114         'jashook',
1115         'JosephTremoulet',
1116         'pgodeq',
1117         'pgavlin',
1118         'rartemev',
1119         'russellhadley',
1120         'RussKeldorph',
1121         'sandreenko',
1122         'sdmaclea',
1123         'swaroop-sridhar',
1124         'jkotas',
1125         'markwilkie',
1126         'weshaggard'
1127     ]
1128
1129     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1130     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1131     def osGroup = getOSGroup(os)
1132     switch (architecture) {
1133         case 'x64': // editor brace matching: {
1134             if (scenario == 'formatting') {
1135                 assert configuration == 'Checked'
1136                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1137                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1138                 }
1139                 break
1140             }
1141
1142             switch (os) {
1143                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1144                 case 'Debian8.4':
1145                 case 'RHEL7.2':
1146                     if (scenario == 'default') {
1147                         assert !isFlowJob
1148                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1149                     }
1150                     break
1151                 case 'Ubuntu16.04':
1152                     assert !isFlowJob
1153                     assert scenario == 'default'
1154                     // Distinguish with the other architectures (arm and x86)
1155                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1156                     break
1157                 case 'Fedora24':
1158                 case 'Ubuntu16.10':
1159                     assert !isFlowJob
1160                     assert scenario == 'default'
1161                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1162                     break
1163                 case 'Ubuntu':
1164                     if (scenario == 'illink') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1166                         break
1167                     }
1168                     // fall through
1169                 case 'OSX10.12':
1170                     // Triggers on the non-flow jobs aren't necessary here
1171                     // Corefx testing uses non-flow jobs.
1172                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1173                         break
1174                     }
1175                     switch (scenario) {
1176                         case 'default':
1177                             // OSX uses checked for default PR tests
1178                             if (configuration == 'Checked') {
1179                                 // Default trigger
1180                                 assert !job.name.contains("centos")
1181                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1182                             }
1183                             break
1184                         case 'jitdiff':
1185                             if (configuration == 'Checked') {
1186                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1187                             }
1188                             break
1189                         case 'ilrt':
1190                             if (configuration == 'Release') {
1191                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1192                             }
1193                             break
1194                         case 'longgc':
1195                             if (configuration == 'Release') {
1196                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1197                             }
1198                             break
1199                         case 'gcsimulator':
1200                             if (configuration == 'Release') {
1201                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1202                             }
1203                             break
1204                         case 'standalone_gc':
1205                             if (configuration == 'Release' || configuration == 'Checked') {
1206                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1207                             }
1208                             break
1209                         case 'gc_reliability_framework':
1210                             if (configuration == 'Release' || configuration == 'Checked') {
1211                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1212                             }
1213                             break
1214                         default:
1215                             if (isJitStressScenario(scenario)) {
1216                                 def displayStr = getStressModeDisplayName(scenario)
1217                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1218                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1219                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1220                             }
1221                             else if (isR2RScenario(scenario)) {
1222                                 if (configuration == 'Release' || configuration == 'Checked') {
1223                                     def displayStr = getR2RDisplayName(scenario)
1224                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1225                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1226                                 }
1227                             }
1228                             else {
1229                                 println("Unknown scenario: ${scenario}");
1230                                 assert false
1231                             }
1232                             break
1233                     }
1234                     break
1235
1236                 case 'CentOS7.1':
1237                     switch (scenario) {
1238                         case 'default':
1239                             // CentOS uses checked for default PR tests while debug is build only
1240                             if (configuration == 'Debug') {
1241                                 // Default trigger
1242                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1243                             }
1244
1245                             // Make sure this is a flow job to get build and test.
1246                             if (configuration == 'Checked' && isFlowJob) {
1247                                 assert job.name.contains("flow")
1248                                 // Default trigger
1249                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1250                             }
1251                             break
1252                         default:
1253                             if (isR2RScenario(scenario)) {
1254                                 if (configuration == 'Release' || configuration == 'Checked') {
1255                                     def displayStr = getR2RDisplayName(scenario)
1256                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1257                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                                 }
1259                             }
1260                             break
1261                     }
1262                     break
1263
1264                 case 'Windows_NT':
1265                     switch (scenario) {
1266                         case 'default':
1267                             // Default trigger
1268                             if (configuration == 'Checked') {
1269                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1270                             }
1271                             break
1272                         case 'jitdiff':
1273                             if (configuration == 'Checked') {
1274                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1275                             }
1276                             break
1277                         case 'ilrt':
1278                             if (configuration == 'Release') {
1279                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1280                             }
1281                             break
1282                         case 'longgc':
1283                             if (configuration == 'Release') {
1284                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1285                             }
1286                             break
1287                         case 'gcsimulator':
1288                             if (configuration == 'Release') {
1289                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1290                             }
1291                             break
1292                         case 'standalone_gc':
1293                             if (configuration == 'Release' || configuration == 'Checked') {
1294                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1295                             }
1296                             break
1297                         case 'gc_reliability_framework':
1298                             if (configuration == 'Release' || configuration == 'Checked') {
1299                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1300                             }
1301                             break
1302                         case 'illink':
1303                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1304                             break
1305                         default:
1306                             if (isJitStressScenario(scenario)) {
1307                                 def displayStr = getStressModeDisplayName(scenario)
1308                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1309                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1310                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1311                             }
1312                             else if (isR2RScenario(scenario)) {
1313                                 if (configuration == 'Release' || configuration == 'Checked') {
1314                                     def displayStr = getR2RDisplayName(scenario)
1315                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1316                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1317                                 }
1318                             }
1319                             else {
1320                                 println("Unknown scenario: ${scenario}");
1321                                 assert false
1322                             }
1323                             break
1324                     }
1325                     break
1326                 default:
1327                     println("Unknown os: ${os}");
1328                     assert false
1329                     break
1330             }
1331             break
1332         // editor brace matching: }
1333         case 'armlb':
1334         case 'arm': // editor brace matching: {
1335             switch (os) {
1336                 case 'Ubuntu':
1337                 case 'Ubuntu16.04':
1338                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1339                         break
1340                     }
1341                     assert scenario == 'default'
1342                     job.with {
1343                         publishers {
1344                             azureVMAgentPostBuildAction {
1345                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1346                             }
1347                         }
1348                     }
1349                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1350                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1351                     break
1352                 case 'Tizen':
1353                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1354                         break
1355                     }
1356                     architecture='armel'
1357                     job.with {
1358                         publishers {
1359                             azureVMAgentPostBuildAction {
1360                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1361                             }
1362                         }
1363                     }
1364                     if (configuration == 'Checked') {
1365                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1366                     }
1367                     else {
1368                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1369                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1370                     }
1371                     break
1372                 case 'Windows_NT':
1373                     // Triggers on the non-flow jobs aren't necessary here
1374                     if (!isFlowJob) {
1375                         break
1376                     }
1377
1378                     // Set up a private trigger
1379                     def contextString = "${os} ${architecture} Cross ${configuration}"
1380                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1381                     if (scenario == 'default') {
1382                         contextString += " Innerloop"
1383                         triggerString += "\\W+Innerloop"
1384                     }
1385                     else {
1386                         contextString += " ${scenario}"
1387                         triggerString += "\\W+${scenario}"
1388                     }
1389
1390                     if (configuration == 'Debug') {
1391                         contextString += " Build"
1392                         triggerString += "\\W+Build"
1393                     } else {
1394                         contextString += " Build and Test"
1395                         triggerString += "\\W+Build and Test"
1396                     }
1397
1398                     triggerString += ".*"
1399
1400                     switch (scenario) {
1401                         case 'default':
1402                             // Only Checked is a default trigger.
1403                             if (configuration == 'Checked')
1404                             {
1405                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1406                             }
1407                             else
1408                             {
1409                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1410                             }
1411                             break
1412                         default:
1413                             // Stress jobs will use this code path.
1414                             if (isArmWindowsScenario(scenario)) {
1415                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1416                             }
1417                             break
1418                     }
1419                     break
1420                 default:
1421                     println("NYI os: ${os}");
1422                     assert false
1423                     break
1424             }
1425             break
1426         // editor brace matching: }
1427         case 'arm64': // editor brace matching: {
1428             // Set up a private trigger
1429             def contextString = "${os} ${architecture} Cross ${configuration}"
1430             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1431             if (scenario == 'default') {
1432                 contextString += " Innerloop"
1433                 triggerString += "\\W+Innerloop"
1434             }
1435             else {
1436                 contextString += " ${scenario}"
1437                 triggerString += "\\W+${scenario}"
1438             }
1439
1440             if (configuration == 'Debug') {
1441                 contextString += " Build"
1442                 triggerString += "\\W+Build"
1443             } else {
1444                 contextString += " Build and Test"
1445                 triggerString += "\\W+Build and Test"
1446             }
1447
1448             triggerString += ".*"
1449
1450             switch (os) {
1451                 case 'Ubuntu':
1452                 case 'Ubuntu16.04':
1453                     switch (scenario) {
1454                         case 'default':
1455                             if (configuration == 'Debug' && !isFlowJob) {
1456                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1457                             }
1458                             else {
1459                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1460                             }
1461
1462                             break
1463                         default:
1464                             if (isR2RScenario(scenario)) {
1465                                 if (configuration == 'Checked' || configuration == 'Release') {
1466                                     def displayStr = getR2RDisplayName(scenario)
1467                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1468                                 }
1469                             }
1470                             break
1471                     }
1472                     break
1473                 case 'Windows_NT':
1474                     // Triggers on the non-flow jobs aren't necessary here
1475                     if (!isFlowJob) {
1476                         break
1477                     }
1478
1479                     assert isArmWindowsScenario(scenario)
1480                     switch (scenario) {
1481                         case 'default':
1482                             if (configuration == 'Checked') {
1483                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1484                             }
1485                             else {
1486                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1487                             }
1488                             break
1489                         default:
1490                             // Stress jobs will use this code path.
1491                             if (isArmWindowsScenario(scenario)) {
1492                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1493                             }
1494                             break
1495                     }
1496                     break
1497                 default:
1498                     println("NYI os: ${os}");
1499                     assert false
1500                     break
1501             }
1502             break
1503         // editor brace matching: }
1504         case 'x86': // editor brace matching: {
1505             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1506             if (os == 'Ubuntu') {
1507                 // Triggers on the non-flow jobs aren't necessary here
1508                 if (!isFlowJob) {
1509                     break
1510                 }
1511                 // on-demand only for ubuntu x86
1512                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1513                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1514                 break
1515             }
1516             switch (scenario) {
1517                 case 'default':
1518                     if (configuration == 'Checked') {
1519                         assert !job.name.contains("centos")
1520                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1521                     }
1522                     else {
1523                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1524                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1525                     }
1526                     break
1527                 case 'ilrt':
1528                     if (configuration == 'Release') {
1529                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1530                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1531                     }
1532                     break
1533                 case 'longgc':
1534                     if (configuration == 'Release') {
1535                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1536                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1537                     }
1538                     break
1539                 case 'gcsimulator':
1540                     if (configuration == 'Release') {
1541                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1542                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1543                     }
1544                     break
1545                 case 'standalone_gc':
1546                     if (configuration == 'Release' || configuration == 'Checked') {
1547                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1548                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1549                     }
1550                     break
1551                 case 'illink':
1552                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1553                     break
1554                 default:
1555                     if (isJitStressScenario(scenario)) {
1556                         def displayStr = getStressModeDisplayName(scenario)
1557                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1558                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1559                     }
1560                     else if (isR2RScenario(scenario)) {
1561                         if (configuration == 'Release' || configuration == 'Checked') {
1562                             def displayStr = getR2RDisplayName(scenario)
1563                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1564                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1565                         }
1566                     }
1567                     else {
1568                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1569                         assert false
1570                     }
1571                     break
1572             }
1573             break
1574          // editor brace matching: }
1575         case 'x64_arm64_altjit':
1576         case 'x86_arm_altjit': // editor brace matching: {
1577             assert (os == 'Windows_NT')
1578             switch (scenario) {
1579                 case 'default':
1580                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1581                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1582                     break
1583                 default:
1584                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1585                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1586                     break
1587             }
1588             break
1589         // editor brace matching: }
1590         default:
1591             println("Unknown architecture: ${architecture}");
1592             assert false
1593             break
1594     }
1595 }
1596
1597 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1598     def buildCommands = [];
1599     def osGroup = getOSGroup(os)
1600     def lowerConfiguration = configuration.toLowerCase()
1601
1602     def priority = '1'
1603     if (scenario == 'default' && isPR == true) {
1604         priority = '0'
1605     }
1606
1607     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1608
1609     def enableCorefxTesting = isCoreFxScenario(scenario)
1610
1611     // Calculate the build steps, archival, and xunit results
1612     switch (os) {
1613         case 'Windows_NT': // editor brace matching: {
1614             switch (architecture) {
1615                 case 'x64':
1616                 case 'x86':
1617                 case 'x86_arm_altjit':
1618                 case 'x64_arm64_altjit':
1619                     def arch = architecture
1620                     def buildOpts = ''
1621                     if (architecture == 'x86_arm_altjit') {
1622                         arch = 'x86'
1623                     }
1624                     else if (architecture == 'x64_arm64_altjit') {
1625                         arch = 'x64'
1626                     }
1627
1628                     if (scenario == 'formatting') {
1629                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1630                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1631                         break
1632                     }
1633
1634                     if (scenario == 'illink') {
1635                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1636                     }
1637
1638                     // If it is a release build for windows, ensure PGO is used, else fail the build
1639                     if ((lowerConfiguration == 'release') &&
1640                         (scenario in Constants.basicScenarios) &&
1641                         (architecture != 'x86_arm_altjit') &&
1642                         (architecture != 'x64_arm64_altjit')) {
1643
1644                         buildOpts += ' -enforcepgo'
1645                     }
1646
1647                     if (enableCorefxTesting) {
1648                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
1649                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
1650                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
1651                         //
1652                         // buildOpts += ' skiptests';
1653                         buildOpts += " -priority=0"
1654                     } else {
1655                         buildOpts += " -priority=${priority}"
1656                     }
1657
1658                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1659                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1660                     // 35 characters long.
1661
1662                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1663
1664                     if (!isBuildOnly) {
1665                         def runtestArguments = ''
1666                         def testOpts = 'collectdumps'
1667
1668                         if (isR2RScenario(scenario)) {
1669
1670                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1671                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1672                             // to cause the tests to be crossgen'ed.
1673
1674                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1675                                 testOpts += ' crossgenaltjit protononjit.dll'
1676                             } else {
1677                                 testOpts += ' crossgen'
1678                             }
1679
1680                             testOpts += ' runcrossgentests'
1681                         }
1682                         else if (scenario == 'jitdiff') {
1683                             testOpts += ' jitdisasm crossgen'
1684                         }
1685                         else if (scenario == 'ilrt') {
1686                             testOpts += ' ilasmroundtrip'
1687                         }
1688                         else if (isLongGc(scenario)) {
1689                             testOpts += " ${scenario} sequential"
1690                         }
1691                         else if (scenario == 'standalone_gc') {
1692                             testOpts += ' gcname clrgc.dll'
1693                         }
1694                         else if (scenario == 'illink') {
1695                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1696                         }
1697
1698                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1699                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1700                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1701                         // which is read by the xunit harness.)
1702                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1703                         {
1704                             def timeout = 1800000
1705                             testOpts += " timeout ${timeout}"
1706                         }
1707
1708                         // If we are running a stress mode, we should write out the set of key
1709                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1710
1711                         def envScriptPath = ''
1712                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1713                             def buildCommandsStr = ''
1714                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1715                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1716
1717                             if (isJitStressScenario(scenario)) {
1718                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1719                             }
1720                             else if (isR2RStressScenario(scenario)) {
1721                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1722                             }
1723
1724                             if (architecture == 'x86_arm_altjit') {
1725                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1726                             }
1727                             else if (architecture == 'x64_arm64_altjit') {
1728                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1729                             }
1730
1731                             envScriptFinalize(os, envScriptPath)
1732
1733                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1734                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1735                             // into a single script.
1736                             buildCommands += buildCommandsStr
1737                         }
1738                         else if (architecture == 'x86_arm_altjit') {
1739                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1740                         }
1741                         else if (architecture == 'x64_arm64_altjit') {
1742                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1743                         }
1744                         if (envScriptPath != '') {
1745                             testOpts += " TestEnv ${envScriptPath}"
1746                         }
1747
1748                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1749
1750                         if (enableCorefxTesting) {
1751                             // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
1752                             // to determine the correct matching corefx version git commit hash.
1753                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1754
1755                             def workspaceRelativeFxRoot = "_/fx"
1756                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1757
1758                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1759
1760                             // Archive and process (only) the test results
1761                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1762                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1763
1764                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1765                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1766                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1767                         }
1768                         else if (isGcReliabilityFramework(scenario)) {
1769                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1770                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1771                         }
1772                         else {
1773                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1774                         }
1775                     }
1776
1777                     if (!enableCorefxTesting) {
1778                         // Run the rest of the build
1779                         // Build the mscorlib for the other OS's
1780                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1781                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1782
1783                         if (arch == "x64") {
1784                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1785                         }
1786
1787                         // Zip up the tests directory so that we don't use so much space/time copying
1788                         // 10s of thousands of files around.
1789                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1790
1791                         if (!isJitStressScenario(scenario)) {
1792                             // For windows, pull full test results and test drops for x86/x64.
1793                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1794                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1795                         }
1796
1797                         if (scenario == 'jitdiff') {
1798                             // retrive jit-dasm output for base commit, and run jit-diff
1799                             if (!isBuildOnly) {
1800                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1801                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1802                             }
1803                         }
1804
1805                         if (!isBuildOnly) {
1806                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1807                         }
1808                     }
1809                     break
1810                 case 'armlb':
1811                 case 'arm':
1812                     assert isArmWindowsScenario(scenario)
1813
1814                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1815                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1816
1817                     def buildArchitecture = 'arm'
1818
1819                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1820                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1821                     def armCrossgenOpt = ''
1822                     if (architecture == 'armlb') {
1823                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1824                     }
1825
1826                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1827                     priority = '1'
1828
1829                     // This is now a build only job. Do not run tests. Use the flow job.
1830                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1831
1832                     // Zip up the tests directory so that we don't use so much space/time copying
1833                     // 10s of thousands of files around.
1834                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1835
1836                     // Add archival.
1837                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1838                     break
1839                 case 'arm64':
1840                     assert isArmWindowsScenario(scenario)
1841
1842                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1843                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1844
1845                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1846                     priority = '1'
1847
1848                     // This is now a build only job. Do not run tests. Use the flow job.
1849                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1850
1851                     // Zip up the tests directory so that we don't use so much space/time copying
1852                     // 10s of thousands of files around.
1853                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1854
1855                     // Add archival.
1856                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1857                     break
1858                 default:
1859                     println("Unknown architecture: ${architecture}");
1860                     assert false
1861                     break
1862             }
1863             break
1864         // editor brace matching: }
1865         case 'Ubuntu':
1866         case 'Ubuntu16.04':
1867         case 'Ubuntu16.10':
1868         case 'Debian8.4':
1869         case 'OSX10.12':
1870         case 'CentOS7.1':
1871         case 'RHEL7.2':
1872         case 'Tizen':
1873         case 'Fedora24': // editor brace matching: {
1874             switch (architecture) {
1875                 case 'x64':
1876                 case 'x86':
1877                     if (architecture == 'x86' && os == 'Ubuntu') {
1878                         // build and PAL test
1879                         def dockerImage = getDockerImageName(architecture, os, true)
1880                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1881                         dockerImage = getDockerImageName(architecture, os, false)
1882                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1883                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1884                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1885                         break
1886                     }
1887
1888                     if (scenario == 'formatting') {
1889                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1890                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1891                         break
1892                     }
1893
1894                     if (scenario == 'illink') {
1895                         assert(os == 'Ubuntu')
1896                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1897                     }
1898
1899                     if (!enableCorefxTesting) {
1900                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1901                         // only on supported OS platforms.
1902                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1903                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1904
1905                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1906                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1907
1908                         // Basic archiving of the build
1909                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1910                         // And pal tests
1911                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1912                     }
1913                     else {
1914                         // Corefx stress testing
1915                         assert os == 'Ubuntu'
1916                         assert architecture == 'x64'
1917                         assert lowerConfiguration == 'checked'
1918                         assert isJitStressScenario(scenario)
1919
1920                         // Build coreclr
1921                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1922
1923                         // Generate the test layout, which restores the corefx package, and allows run-corefx-tests.py to determine the
1924                         // matching corefx version git commit hash.
1925                         buildCommands += "./build-test.sh ${architecture} ${lowerConfiguration} generatelayoutonly"
1926
1927                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1928
1929                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1930                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1931                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1932                         buildCommands += envScriptCmds
1933
1934                         // Build and text corefx
1935                         def workspaceRelativeFxRoot = "_/fx"
1936                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1937
1938                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1939
1940                         // Archive and process (only) the test results
1941                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1942                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1943                     }
1944                     break
1945                 case 'arm64':
1946                     if (!enableCorefxTesting) {
1947                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1948
1949                         // HACK -- Arm64 does not have corefx jobs yet.
1950                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1951                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1952                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1953                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1954
1955                         // Basic archiving of the build
1956                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1957                     }
1958                     break
1959                 case 'arm':
1960                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1961                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1962
1963                     // default values for Ubuntu
1964                     def arm_abi="arm"
1965                     def linuxCodeName="trusty"
1966                     if (os == 'Ubuntu16.04') {
1967                         linuxCodeName="xenial"
1968                     }
1969                     else if (os == 'Tizen') {
1970                         arm_abi="armel"
1971                         linuxCodeName="tizen"
1972                     }
1973
1974                     // Unzip the Windows test binaries first. Exit with 0
1975                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1976
1977                     // Unpack the corefx binaries
1978                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1979                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1980                     if (os != 'Tizen') {
1981                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1982                     }
1983                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1984                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1985                     if (os == 'Ubuntu') {
1986                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1987                     }
1988
1989                     // Call the ARM CI script to cross build and test using docker
1990                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1991                     --mode=docker \\
1992                     --${arm_abi} \\
1993                     --linuxCodeName=${linuxCodeName} \\
1994                     --buildConfig=${lowerConfiguration} \\
1995                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1996                     --coreFxBinDir=./bin/CoreFxBinDir \\
1997                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1998
1999                     // Basic archiving of the build, no pal tests
2000                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2001                     break
2002                 default:
2003                     println("Unknown architecture: ${architecture}");
2004                     assert false
2005                     break
2006             }
2007             break
2008         // editor brace matching: }
2009         default:
2010             println("Unknown os: ${os}");
2011             assert false
2012             break
2013     } // os
2014
2015     return buildCommands
2016 }
2017
2018 Constants.allScenarios.each { scenario ->
2019     [true, false].each { isPR ->
2020         Constants.architectureList.each { architecture ->
2021             Constants.configurationList.each { configuration ->
2022                 Constants.osList.each { os ->
2023                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2024                     // and reset the os to Windows_NT
2025                     def isBuildOnly = false
2026                     if (os == 'Windows_NT_BuildOnly') {
2027                         isBuildOnly = true
2028                         os = 'Windows_NT'
2029                     }
2030
2031                     // Tizen is only supported for arm architecture
2032                     if (os == 'Tizen' && architecture != 'arm') {
2033                         return
2034                     }
2035
2036                     // Skip totally unimplemented (in CI) configurations.
2037                     switch (architecture) {
2038                         case 'arm64':
2039                             if (os == 'Ubuntu16.04') {
2040                                 os = 'Ubuntu'
2041                             }
2042
2043                             // Windows and Ubuntu only
2044                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2045                                 return
2046                             }
2047                             break
2048                         case 'arm':
2049                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2050                                 return
2051                             }
2052                             break
2053                         case 'armlb':
2054                             if (os != 'Windows_NT') {
2055                                 return
2056                             }
2057                             break
2058                         case 'x86':
2059                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2060                                 return
2061                             }
2062                             break
2063                         case 'x86_arm_altjit':
2064                         case 'x64_arm64_altjit':
2065                             if (os != 'Windows_NT') {
2066                                 return
2067                             }
2068                             break
2069                         case 'x64':
2070                             // Everything implemented
2071                             break
2072                         default:
2073                             println("Unknown architecture: ${architecture}")
2074                             assert false
2075                             break
2076                     }
2077
2078                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2079                     // with checked builds)
2080                     if (isJitStressScenario(scenario)) {
2081                         if (configuration != 'Checked') {
2082                             return
2083                         }
2084
2085                         // Since these are just execution time differences,
2086                         // skip platforms that don't execute the tests here (Windows_NT only)
2087                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2088                         if (!isEnabledOS || isBuildOnly) {
2089                             return
2090                         }
2091
2092                         switch (architecture) {
2093                             case 'x64':
2094                             case 'x86':
2095                             case 'x86_arm_altjit':
2096                             case 'x64_arm64_altjit':
2097                                 // x86 ubuntu: default only
2098                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2099                                     return
2100                                 }
2101                                 // Windows: Everything implemented
2102                                 break
2103
2104                             default:
2105                                 // arm, arm64, armlb: stress is handled through flow jobs.
2106                                 return
2107                         }
2108                     }
2109                     else if (isR2RScenario(scenario)) {
2110                         if (os != 'Windows_NT') {
2111                             return
2112                         }
2113                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2114                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2115                             return
2116                         }
2117                     }
2118                     else {
2119                         // Skip scenarios
2120                         switch (scenario) {
2121                             case 'ilrt':
2122                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2123                                 // the default scenario build
2124                                 if (os != 'Windows_NT') {
2125                                     return
2126                                 }
2127                                 // Only x64 for now
2128                                 if (architecture != 'x64') {
2129                                     return
2130                                 }
2131                                 // Release only
2132                                 if (configuration != 'Release') {
2133                                     return
2134                                 }
2135                                 break
2136                             case 'jitdiff':
2137                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2138                                     return
2139                                 }
2140                                 if (architecture != 'x64') {
2141                                     return
2142                                 }
2143                                 if (configuration != 'Checked') {
2144                                     return
2145                                 }
2146                                 break
2147                             case 'longgc':
2148                             case 'gcsimulator':
2149                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2150                                     return
2151                                 }
2152                                 if (architecture != 'x64') {
2153                                     return
2154                                 }
2155                                 if (configuration != 'Release') {
2156                                     return
2157                                 }
2158                                 break
2159                             case 'gc_reliability_framework':
2160                             case 'standalone_gc':
2161                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2162                                     return
2163                                 }
2164
2165                                 if (architecture != 'x64') {
2166                                     return
2167                                 }
2168
2169                                 if (configuration != 'Release' && configuration != 'Checked') {
2170                                     return
2171                                 }
2172                                 break
2173                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2174                             case 'formatting':
2175                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2176                                     return
2177                                 }
2178                                 if (architecture != 'x64') {
2179                                     return
2180                                 }
2181                                 if (configuration != 'Checked') {
2182                                     return
2183                                 }
2184                                 if (isBuildOnly) {
2185                                     return
2186                                 }
2187                                 break
2188                             case 'illink':
2189                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2190                                     return
2191                                 }
2192                                 if (architecture != 'x64' && architecture != 'x86') {
2193                                     return
2194                                 }
2195                                 if (isBuildOnly) {
2196                                     return
2197                                 }
2198                                 break
2199                             case 'default':
2200                                 // Nothing skipped
2201                                 break
2202                             default:
2203                                 println("Unknown scenario: ${scenario}")
2204                                 assert false
2205                                 break
2206                         }
2207                     }
2208
2209                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2210                     // runtime behavior, not compile-time behavior, are not interesting.
2211                     switch (architecture) {
2212                         case 'x86_arm_altjit':
2213                         case 'x64_arm64_altjit':
2214                             if (isGCStressRelatedTesting(scenario)) {
2215                                 return
2216                             }
2217                             break
2218                         default:
2219                             break
2220                     }
2221
2222                     // Calculate names
2223                     def lowerConfiguration = configuration.toLowerCase()
2224                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2225                     def folderName = getJobFolder(scenario)
2226
2227                     // Create the new job
2228                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2229                     addToViews(newJob, isPR, architecture, os)
2230
2231                     def machineAffinityOptions = null
2232
2233                     if (os != 'Windows_NT') {
2234                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2235                     }
2236                     else {
2237                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2238                     }
2239
2240                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2241
2242                     // Add all the standard options
2243                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2244                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2245
2246                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2247                     def osGroup = getOSGroup(os)
2248
2249                     newJob.with {
2250                         steps {
2251                             if (os == 'Windows_NT') {
2252                                 buildCommands.each { buildCommand ->
2253                                     batchFile(buildCommand)
2254                                 }
2255                             }
2256                             else {
2257                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2258                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2259                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2260                                     // Define the Windows Tests and Corefx build job names
2261                                     def WindowsTestsName = projectFolder + '/' +
2262                                                            Utilities.getFullJobName(project,
2263                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2264                                                                                     false)
2265                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2266                                                        Utilities.getFolderName(branch)
2267
2268                                     // Copy the Windows test binaries and the Corefx build binaries
2269                                     copyArtifacts(WindowsTestsName) {
2270                                         includePatterns('bin/tests/tests.zip')
2271                                         buildSelector {
2272                                             latestSuccessful(true)
2273                                         }
2274                                     }
2275
2276                                     def arm_abi = 'arm'
2277                                     def corefx_os = 'linux'
2278                                     if (os == 'Tizen') {
2279                                         arm_abi = 'armel'
2280                                         corefx_os = 'tizen'
2281                                     }
2282
2283                                     // Let's use release CoreFX to test checked CoreCLR,
2284                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2285                                     def corefx_lowerConfiguration = lowerConfiguration
2286                                     if ( lowerConfiguration == 'checked' ) {
2287                                         corefx_lowerConfiguration='release'
2288                                     }
2289
2290                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2291                                         includePatterns('bin/build.tar.gz')
2292                                         buildSelector {
2293                                             latestSuccessful(true)
2294                                         }
2295                                     }
2296                                 }
2297
2298                                 buildCommands.each { buildCommand ->
2299                                     shell(buildCommand)
2300                                 }
2301                             }
2302                         }
2303                     } // newJob.with
2304
2305                 } // os
2306             } // configuration
2307         } // architecture
2308     } // isPR
2309 } // scenario
2310
2311
2312 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2313 Constants.allScenarios.each { scenario ->
2314     [true, false].each { isPR ->
2315         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2316             Constants.crossList.each { os ->
2317                 if (architecture == 'arm64') {
2318                     if (os != "Ubuntu" && os != "Windows_NT") {
2319                         return
2320                     }
2321                 } else if (architecture == 'arm' || architecture == 'armlb') {
2322                     if (os != 'Windows_NT') {
2323                         return
2324                     }
2325                 }
2326                 else if (architecture == 'x86') {
2327                     if (os != "Ubuntu") {
2328                         return
2329                     }
2330                 }
2331
2332                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2333
2334                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2335                     return
2336                 }
2337
2338                 Constants.configurationList.each { configuration ->
2339
2340                     // First, filter based on OS.
2341
2342                     if (os == 'Windows_NT') {
2343                         if (!isArmWindowsScenario(scenario)) {
2344                             return
2345                         }
2346                     }
2347                     else {
2348                         // Non-Windows
2349                         if (architecture == 'arm64') {
2350                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2351                                 return
2352                             }
2353                         }
2354                         else if (architecture == 'x86') {
2355                             // Linux/x86 only want default test
2356                             if (scenario != 'default') {
2357                                 return
2358                             }
2359                         }
2360                     }
2361
2362                     // For CentOS, we only want Checked/Release builds.
2363                     if (os == 'CentOS7.1') {
2364                         if (configuration != 'Checked' && configuration != 'Release') {
2365                             return
2366                         }
2367                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2368                             return
2369                         }
2370                     }
2371
2372                     // For RedHat and Debian, we only do Release builds.
2373                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2374                         if (configuration != 'Release') {
2375                             return
2376                         }
2377                         if (scenario != 'default') {
2378                             return
2379                         }
2380                     }
2381
2382                     // Next, filter based on scenario.
2383
2384                     if (isJitStressScenario(scenario)) {
2385                         if (configuration != 'Checked') {
2386                             return
2387                         }
2388                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2389                         if (isCoreFxScenario(scenario)) {
2390                             return
2391                         }
2392                     }
2393                     else if (isR2RBaselineScenario(scenario)) {
2394                         if (configuration != 'Checked' && configuration != 'Release') {
2395                             return
2396                         }
2397                     }
2398                     else if (isR2RStressScenario(scenario)) {
2399                         if (configuration != 'Checked') {
2400                             return
2401                         }
2402                     }
2403                     else {
2404                         // Skip scenarios
2405                         switch (scenario) {
2406                             case 'ilrt':
2407                             case 'longgc':
2408                             case 'gcsimulator':
2409                                 // Long GC tests take a long time on non-Release builds
2410                                 // ilrt is also Release only
2411                                 if (configuration != 'Release') {
2412                                     return
2413                                 }
2414                                 break
2415                             case 'jitdiff':
2416                                 if (configuration != 'Checked') {
2417                                     return;
2418                                 }
2419                                 break
2420                             case 'gc_reliability_framework':
2421                             case 'standalone_gc':
2422                                 if (configuration != 'Release' && configuration != 'Checked') {
2423                                     return
2424                                 }
2425                                 break
2426                             case 'formatting':
2427                                 return
2428                             case 'illink':
2429                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2430                                     return
2431                                 }
2432                                 break
2433                             case 'default':
2434                                 // Nothing skipped
2435                                 break
2436                             default:
2437                                 println("Unknown scenario: ${scenario}")
2438                                 assert false
2439                                 break
2440                         }
2441                     }
2442
2443                     // Done filtering. Now, create the jobs.
2444
2445                     def lowerConfiguration = configuration.toLowerCase()
2446                     def osGroup = getOSGroup(os)
2447                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2448
2449                     def inputCoreCLRBuildName = projectFolder + '/' +
2450                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2451
2452                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2453                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2454                     def testBuildScenario = scenario
2455                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2456                         testBuildScenario = 'default'
2457                     }
2458
2459                     def inputWindowsTestBuildArch = architecture
2460                     if (architecture == "arm64" && os != "Windows_NT") {
2461                         // Use the x64 test build for arm64 unix
2462                         inputWindowsTestBuildArch = "x64"
2463                     }
2464
2465                     def inputWindowsTestsBuildName = ""
2466
2467                     if (isJitStressScenario(scenario)) {
2468                         inputWindowsTestsBuildName = projectFolder + '/' +
2469                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2470                     } else {
2471                         inputWindowsTestsBuildName = projectFolder + '/' +
2472                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2473
2474                     }
2475
2476
2477                     // Enable Server GC for Ubuntu PR builds
2478                     def serverGCString = ''
2479                     if (os == 'Ubuntu' && isPR) {
2480                         serverGCString = '--useServerGC'
2481                     }
2482
2483                     def testOpts = ''
2484
2485                     if (isR2RScenario(scenario)) {
2486
2487                         testOpts += ' --crossgen --runcrossgentests'
2488
2489                         if (scenario == 'r2r_jitstress1') {
2490                             testOpts += ' --jitstress=1'
2491                         }
2492                         else if (scenario == 'r2r_jitstress2') {
2493                             testOpts += ' --jitstress=2'
2494                         }
2495                         else if (scenario == 'r2r_jitstressregs1') {
2496                             testOpts += ' --jitstressregs=1'
2497                         }
2498                         else if (scenario == 'r2r_jitstressregs2') {
2499                             testOpts += ' --jitstressregs=2'
2500                         }
2501                         else if (scenario == 'r2r_jitstressregs3') {
2502                             testOpts += ' --jitstressregs=3'
2503                         }
2504                         else if (scenario == 'r2r_jitstressregs4') {
2505                             testOpts += ' --jitstressregs=4'
2506                         }
2507                         else if (scenario == 'r2r_jitstressregs8') {
2508                             testOpts += ' --jitstressregs=8'
2509                         }
2510                         else if (scenario == 'r2r_jitstressregs0x10') {
2511                             testOpts += ' --jitstressregs=0x10'
2512                         }
2513                         else if (scenario == 'r2r_jitstressregs0x80') {
2514                             testOpts += ' --jitstressregs=0x80'
2515                         }
2516                         else if (scenario == 'r2r_jitstressregs0x1000') {
2517                             testOpts += ' --jitstressregs=0x1000'
2518                         }
2519                         else if (scenario == 'r2r_jitminopts') {
2520                             testOpts += ' --jitminopts'
2521                         }
2522                         else if (scenario == 'r2r_jitforcerelocs') {
2523                             testOpts += ' --jitforcerelocs'
2524                         }
2525                         else if (scenario == 'r2r_gcstress15') {
2526                             testOpts += ' --gcstresslevel=0xF'
2527                         }
2528                     }
2529                     else if (scenario == 'jitdiff') {
2530                         testOpts += ' --jitdisasm --crossgen'
2531                     }
2532                     else if (scenario == 'illink') {
2533                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2534                     }
2535                     else if (isLongGc(scenario)) {
2536                         // Long GC tests behave very poorly when they are not
2537                         // the only test running (many of them allocate until OOM).
2538                         testOpts += ' --sequential'
2539
2540                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2541                         // for running long GC and GCSimulator tests, respectively. We don't use them
2542                         // here because using a playlist file produces much more readable output on the CI machines
2543                         // and reduces running time.
2544                         //
2545                         // The Long GC playlist contains all of the tests that are
2546                         // going to be run. The GCSimulator playlist contains all of
2547                         // the GC simulator tests.
2548                         if (scenario == 'longgc') {
2549                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2550                         }
2551                         else if (scenario == 'gcsimulator') {
2552                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2553                         }
2554                     }
2555                     else if (isGcReliabilityFramework(scenario)) {
2556                         testOpts += ' --build-overlay-only'
2557                     }
2558                     else if (scenario == 'standalone_gc') {
2559                         if (osGroup == 'OSX') {
2560                             testOpts += ' --gcname=libclrgc.dylib'
2561                         }
2562                         else if (osGroup == 'Linux') {
2563                             testOpts += ' --gcname=libclrgc.so'
2564                         }
2565                         else {
2566                             println("Unexpected OS group: ${osGroup} for os ${os}")
2567                             assert false
2568                         }
2569                     }
2570
2571                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2572
2573                     def folder = getJobFolder(scenario)
2574                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2575                         // Add parameters for the inputs
2576
2577                         if (windowsArmJob == true) {
2578                             parameters {
2579                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2580                             }
2581                         }
2582                         else {
2583                             parameters {
2584                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2585                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2586                             }
2587                         }
2588
2589                         steps {
2590                             // Set up the copies
2591
2592                             // Coreclr build containing the tests and mscorlib
2593                             // pri1 jobs still need to copy windows_nt built tests
2594                             if (windowsArmJob != true) {
2595                                 copyArtifacts(inputWindowsTestsBuildName) {
2596                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2597                                     buildSelector {
2598                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2599                                     }
2600                                 }
2601                             }
2602
2603                             // Coreclr build we are trying to test
2604                             //
2605                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2606
2607                             copyArtifacts(inputCoreCLRBuildName) {
2608                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2609                                 buildSelector {
2610                                     buildNumber('${CORECLR_BUILD}')
2611                                 }
2612                             }
2613
2614                             // Windows CoreCLR Arm(64) will restore corefx
2615                             // packages correctly.
2616                             //
2617                             // In addition, test steps are entirely different
2618                             // because we do not have a unified runner
2619                             if (windowsArmJob != true) {
2620                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2621
2622                                 // HACK -- Arm64 does not have corefx jobs yet.
2623                                 // Clone corefx and build the native packages overwriting the x64 packages.
2624                                 if (architecture == 'arm64') {
2625                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2626                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2627                                 }
2628                                 else if (architecture == 'x86') {
2629                                     shell("mkdir ./bin/CoreFxNative")
2630
2631                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2632                                         includePatterns('bin/build.tar.gz')
2633                                         targetDirectory('bin/CoreFxNative')
2634                                         buildSelector {
2635                                             latestSuccessful(true)
2636                                         }
2637                                     }
2638
2639                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2640                                 }
2641
2642                                 // Unzip the tests first.  Exit with 0
2643                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2644                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2645
2646                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2647
2648                                 // Execute the tests
2649                                 def runDocker = isNeedDocker(architecture, os, false)
2650                                 def dockerPrefix = ""
2651                                 def dockerCmd = ""
2652                                 if (runDocker) {
2653                                     def dockerImage = getDockerImageName(architecture, os, false)
2654                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2655                                     dockerCmd = dockerPrefix + "${dockerImage} "
2656                                 }
2657
2658                                 // If we are running a stress mode, we'll set those variables first
2659                                 def testEnvOpt = ""
2660                                 if (isJitStressScenario(scenario)) {
2661                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2662                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2663                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2664                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2665                                     shell("${envScriptCmds}")
2666                                     testEnvOpt = "--test-env=" + scriptFileName
2667                                 }
2668
2669                                 if (isGCStressRelatedTesting(scenario)) {
2670                                     shell('./init-tools.sh')
2671                                 }
2672
2673                                 shell("""${dockerCmd}./tests/runtest.sh \\
2674                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2675                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2676                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2677                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2678
2679                                 if (isGcReliabilityFramework(scenario)) {
2680                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2681                                     if (serverGCString != '') {
2682                                         if (runDocker) {
2683                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2684                                         }
2685                                         else {
2686                                             shell("export COMPlus_gcServer=1")
2687                                         }
2688                                     }
2689
2690                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2691                                 }
2692                             }
2693
2694                             else { // windowsArmJob == true
2695                                 // Unzip tests.
2696                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2697
2698                                 // Build the build commands
2699                                 def buildCommands = ""
2700
2701                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2702                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2703                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2704
2705                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2706                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2707
2708                                 // For all jobs
2709                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2710
2711                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2712                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2713
2714                                 // ARM legacy backend; this is an altjit.
2715                                 if (architecture == "armlb") {
2716                                     addEnvVariable("COMPlus_AltJit", "*")
2717                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2718                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2719                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2720                                 }
2721
2722                                 // If we are running a stress mode, we'll set those variables as well
2723                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2724                                     def stressValues = null
2725                                     if (isJitStressScenario(scenario)) {
2726                                         stressValues = Constants.jitStressModeScenarios[scenario]
2727                                     }
2728                                     else {
2729                                         stressValues = Constants.r2rStressScenarios[scenario]
2730                                     }
2731
2732                                     stressValues.each { key, value ->
2733                                         addEnvVariable(key, value)
2734                                     }
2735                                 }
2736
2737                                 if (isR2RScenario(scenario)) {
2738                                     // Crossgen the framework assemblies.
2739                                     buildCommands += """
2740 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2741 @goto skip_PrecompileAssembly
2742
2743 :PrecompileAssembly
2744 @REM Skip mscorlib since it is already precompiled.
2745 @if /I "%3" == "mscorlib.dll" exit /b 0
2746 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2747
2748 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2749 @if "%errorlevel%" == "-2146230517" (
2750     echo %2 is not a managed assembly.
2751 ) else if "%errorlevel%" == "-2146234344" (
2752     echo %2 is not a managed assembly.
2753 ) else if %errorlevel% neq 0 (
2754     echo Unable to precompile %2
2755 ) else (
2756     echo Precompiled %2
2757 )
2758 @exit /b 0
2759
2760 :skip_PrecompileAssembly
2761 """
2762
2763                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2764                                     // crossgen on tests before running them.
2765                                     addEnvVariable("RunCrossGen", "true")
2766                                 }
2767
2768                                 // Create the smarty command
2769                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2770                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2771                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2772
2773                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2774
2775                                 if (architecture == "armlb") {
2776                                     addExclude("LEGACYJIT_FAIL")
2777                                 }
2778
2779                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2780                                     def failTag = "JITSTRESS_FAIL"
2781                                     def excludeTag = "JITSTRESS_EXCLUDE"
2782
2783                                     if (scenario.contains('gc')) {
2784                                         failTag = "GCSTRESS_FAIL"
2785                                         excludeTag = "GCSTRESS_EXCLUDE"
2786                                     }
2787
2788                                     addArchSpecificExclude(architecture, failTag)
2789                                     addArchSpecificExclude(architecture, excludeTag)
2790                                 }
2791                                 else {
2792                                     addExclude("pri1")
2793                                 }
2794
2795                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2796                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2797                                 addExclude("LONG_RUNNING")
2798
2799                                 smartyCommand += "/lstFile Tests.lst"
2800
2801                                 def testListArch = [
2802                                     'arm64': 'arm64',
2803                                     'arm': 'arm',
2804                                     'armlb': 'arm'
2805                                 ]
2806
2807                                 def archLocation = testListArch[architecture]
2808
2809                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2810                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2811                                 addCommand("${smartyCommand}")
2812
2813                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2814                                 // However, we also need to remove all the variables that were set during this batch file, so we
2815                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2816                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2817                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2818                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2819                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2820                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2821
2822                                 // ZIP up the smarty output, no matter what the smarty result.
2823                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2824
2825                                 addCommand("echo %errorlevel%")
2826                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2827
2828                                 // Use the smarty errorlevel as the script errorlevel.
2829                                 addCommand("exit /b %__save_smarty_errorlevel%")
2830
2831                                 batchFile(buildCommands)
2832                             }
2833                         }
2834                     }
2835
2836                     addToViews(newJob, isPR, architecture, os)
2837
2838                     if (scenario == 'jitdiff') {
2839                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2840                     }
2841
2842                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2843                     if (os in ['Ubuntu']) {
2844                         SummaryBuilder summaries = new SummaryBuilder()
2845                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2846                         summaries.emit(newJob)
2847                     }
2848
2849                     def affinityOptions = null
2850
2851                     if (windowsArmJob == true) {
2852                         affinityOptions = [
2853                             "use_arm64_build_machine" : false
2854                         ]
2855                     }
2856
2857                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2858                         affinityOptions = [
2859                             "large_pages" : false
2860                         ]
2861                     }
2862
2863                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2864                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2865
2866                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2867
2868                     if (windowsArmJob != true) {
2869                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2870                     }
2871                     else {
2872                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2873
2874                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2875                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2876                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2877                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2878                     }
2879
2880                     // Create a build flow to join together the build and tests required to run this test.
2881                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2882                     // Linux CoreCLR test
2883                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2884                     def fullTestJobName = projectFolder + '/' + newJob.name
2885                     // Add a reference to the input jobs for report purposes
2886                     JobReport.Report.addReference(inputCoreCLRBuildName)
2887                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2888                     JobReport.Report.addReference(fullTestJobName)
2889                     def newFlowJob = null
2890
2891                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2892                         // Do not create the flow job for RHEL jobs.
2893                         return
2894                     }
2895
2896                     // For pri0 jobs we can build tests on unix
2897                     if (windowsArmJob) {
2898                         // For Windows arm jobs there is no reason to build a parallel test job.
2899                         // The product build supports building and archiving the tests.
2900
2901                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2902                         buildFlow("""
2903 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2904
2905 // And then build the test build
2906 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2907 """)
2908                         }
2909                     }
2910                     else {
2911                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2912                         buildFlow("""
2913 // Build the input jobs in parallel
2914 parallel (
2915 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2916 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2917 )
2918
2919 // And then build the test build
2920 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2921             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2922 """)
2923                         }
2924                     }
2925
2926                     addToViews(newFlowJob, isPR, architecture, os)
2927
2928                     // For the flow jobs set the machine affinity as x64 if an armarch.
2929                     def flowArch = architecture
2930
2931                     if (flowArch in validWindowsNTCrossArches) {
2932                         flowArch = 'x64'
2933                         affinityOptions = null
2934                     }
2935
2936                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2937                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2938                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2939                 } // configuration
2940             } // os
2941         } // architecture
2942     } // isPR
2943 } // scenario
2944
2945 JobReport.Report.generateJobReport(out)
2946
2947 // Make the call to generate the help job
2948 Utilities.createHelperJob(this, project, branch,
2949     "Welcome to the ${project} Repository",  // This is prepended to the help message
2950     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2951
2952 Utilities.addCROSSCheck(this, project, branch)