netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'jitincompletehwintrinsic'
 266                // 'jitx86hwintrinsicnoavx'
 267                // 'jitx86hwintrinsicnoavx2'
 268                // 'jitx86hwintrinsicnosimd'
 269                // 'jitnox86hwintrinsic'
 270                'corefx_baseline':                        [],    // corefx tests don't use smarty
 271                'corefx_minopts':                         [],    // corefx tests don't use smarty
 272                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 273                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 274                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 275                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 276                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 277                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 278                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 279                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 280                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 281                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 283                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 284                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 285                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 286                'heapverify1':                            [],
 287                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 288                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 290                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 293
 294                //
 295                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 296                //
 297
 298                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 299                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 317     ]
 318
 319     def static validLinuxArm64Scenarios = [
 320         'normal',
 321         'r2r',
 322         'innerloop',
 323         'gcstress0x3',
 324         'gcstress0xc'
 325     ]
 326
 327     def static configurationList = ['Debug', 'Checked', 'Release']
 328
 329     // This is the set of architectures
 330     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 331 }
 332
 333 // **************************************************************
 334 // Create some specific views
 335 //
 336 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 337 // views based on a single regular expression. These views will be generated by adding a
 338 // specific set of jobs to them.
 339 //
 340 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 341 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 342 // those are required, don't add them (which simplifies the view pages, as well).
 343 // **************************************************************
 344
 345 class Views {
 346     def static MergeJobView = null
 347     def static PeriodicJobView = null
 348     def static ArchitectureViews = [:]
 349     def static OSViews = [:]
 350 }
 351
 352 // MergeJobView: include all jobs that execute when a PR change is merged.
 353 Views.MergeJobView = listView('Merge') {
 354     recurse()
 355     columns {
 356         status()
 357         weather()
 358         name()
 359         lastSuccess()
 360         lastFailure()
 361         lastDuration()
 362         buildButton()
 363     }
 364 }
 365
 366 // PeriodicJobView: include all jobs that execute on a schedule
 367 Views.PeriodicJobView = listView('Periodic') {
 368     recurse()
 369     columns {
 370         status()
 371         weather()
 372         name()
 373         lastSuccess()
 374         lastFailure()
 375         lastDuration()
 376         buildButton()
 377     }
 378 }
 379
 380 // Create a view for non-PR jobs for each architecture.
 381 Constants.architectureList.each { architecture ->
 382     Views.ArchitectureViews[architecture] = listView(architecture) {
 383         recurse()
 384         columns {
 385             status()
 386             weather()
 387             name()
 388             lastSuccess()
 389             lastFailure()
 390             lastDuration()
 391             buildButton()
 392         }
 393     }
 394 }
 395
 396 // Create a view for non-PR jobs for each OS.
 397 Constants.osList.each { os ->
 398     // Don't create one for the special 'Windows_NT_BuildOnly'
 399     if (os == 'Windows_NT_BuildOnly') {
 400         return
 401     }
 402     Views.OSViews[os] = listView(os) {
 403         recurse()
 404         columns {
 405             status()
 406             weather()
 407             name()
 408             lastSuccess()
 409             lastFailure()
 410             lastDuration()
 411             buildButton()
 412         }
 413     }
 414 }
 415
 416 def static addToMergeView(def job) {
 417     Views.MergeJobView.with {
 418         jobs {
 419             name(job.name)
 420         }
 421     }
 422 }
 423
 424 def static addToPeriodicView(def job) {
 425     Views.PeriodicJobView.with {
 426         jobs {
 427             name(job.name)
 428         }
 429     }
 430 }
 431
 432 def static addToViews(def job, def isPR, def architecture, def os) {
 433     if (isPR) {
 434         // No views want PR jobs currently.
 435         return
 436     }
 437
 438     // Add to architecture view.
 439     Views.ArchitectureViews[architecture].with {
 440         jobs {
 441             name(job.name)
 442         }
 443     }
 444
 445     // Add to OS view.
 446     Views.OSViews[os].with {
 447         jobs {
 448             name(job.name)
 449         }
 450     }
 451 }
 452
 453 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 454     addToPeriodicView(job)
 455     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 456 }
 457
 458 def static addGithubPushTriggerHelper(def job) {
 459     addToMergeView(job)
 460     Utilities.addGithubPushTrigger(job)
 461 }
 462
 463
 464 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 465     assert os instanceof String
 466     assert architecture instanceof String
 467
 468     def armArches = ['arm', 'armlb', 'arm64']
 469     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 470
 471     if (!(architecture in armArches)) {
 472         assert options == null
 473         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 474
 475         return
 476     }
 477
 478     // This is an arm(64) job.
 479     //
 480     // There are several options.
 481     //
 482     // Windows_NT
 483     //
 484     // Arm32 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 486     // Arm32 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 488     //
 489     // Arm64 (Build) -> latest-arm64
 490     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 491     // Arm64 (Test)  -> arm64-windows_nt
 492     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 493     //
 494     // Ubuntu
 495     //
 496     // Arm32 (Build) -> arm-cross-latest
 497     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 498     // Arm32 (Test)  -> NYI Arch not supported
 499     //       |->
 500     //
 501     // Arm64 (Build) -> arm64-cross-latest
 502     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 503     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 504     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 505     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 506     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 507
 508     // This has to be a arm arch
 509     assert architecture in armArches
 510     if (os == "Windows_NT") {
 511         // Arm(64) Windows jobs share the same machines for now
 512         def isBuild = options['use_arm64_build_machine'] == true
 513
 514         if (isBuild == true) {
 515             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 516         } else {
 517             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 518         }
 519     } else {
 520         assert os != 'Windows_NT'
 521         assert os in supportedArmLinuxOs
 522
 523         if (architecture == 'arm' || architecture == 'armlb') {
 524             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 525         } else {
 526             // Arm64 Linux
 527             if (options['is_build_only'] == true) {
 528                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 529             } else {
 530                 // Arm64 Test Machines
 531                 if (options['large_pages'] == false) {
 532                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 533                 } else {
 534                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 535                 }
 536             }
 537         }
 538     }
 539 }
 540
 541 def static isGCStressRelatedTesting(def scenario) {
 542     // The 'r2r_gcstress15' scenario is a basic scenario.
 543     // Detect it and make it a GCStress related.
 544     if (scenario == 'r2r_gcstress15')
 545     {
 546         return true;
 547     }
 548
 549     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 550     def scenarioName = scenario.toLowerCase()
 551     def isGCStressTesting = false
 552     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 553         if (k in gcStressTestEnvVars) {
 554             isGCStressTesting = true;
 555         }
 556     }
 557     return isGCStressTesting
 558 }
 559
 560 def static isCoreFxScenario(def scenario) {
 561     def corefx_prefix = 'corefx_'
 562     if (scenario.length() < corefx_prefix.length()) {
 563         return false
 564     }
 565     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 566 }
 567
 568 def static isR2RBaselineScenario(def scenario) {
 569     return (scenario == 'r2r')
 570 }
 571
 572 def static isR2RStressScenario(def scenario) {
 573     return Constants.r2rStressScenarios.containsKey(scenario)
 574 }
 575
 576 def static isR2RScenario(def scenario) {
 577     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 578 }
 579
 580 def static isJitStressScenario(def scenario) {
 581     return Constants.jitStressModeScenarios.containsKey(scenario)
 582 }
 583
 584 def static isLongGc(def scenario) {
 585     return (scenario == 'longgc' || scenario == 'gcsimulator')
 586 }
 587
 588 def static isJitDiff(def scenario) {
 589     return (scenario == 'jitdiff')
 590 }
 591
 592 def static isGcReliabilityFramework(def scenario) {
 593     return (scenario == 'gc_reliability_framework')
 594 }
 595
 596 def static isArmWindowsScenario(def scenario) {
 597     return Constants.validArmWindowsScenarios.containsKey(scenario)
 598 }
 599
 600 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 601     if (isBuildOnly == true) {
 602         os = 'Windows_NT_buildOnly'
 603     }
 604
 605     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 606
 607     if (validOsPrTriggerArchConfigs == null) {
 608         return false
 609     }
 610
 611     if (validOsPrTriggerArchConfigs[architecture] != null) {
 612         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 613
 614         if (!(configuration in validOsPrTriggerConfigs)) {
 615             return false
 616         }
 617     } else {
 618         return false
 619     }
 620
 621     return true
 622 }
 623
 624 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 625     // 2 hours (120 minutes) is the default timeout
 626     def timeout = 120
 627     def innerLoop = (scenario == "innerloop")
 628
 629     if (!innerLoop) {
 630         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 631         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 632         timeout = 240
 633     }
 634
 635     if (!isBuildOnly) {
 636         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 637         if (isGCStressRelatedTesting(scenario)) {
 638             timeout = 4320
 639         }
 640         else if (isCoreFxScenario(scenario)) {
 641             timeout = 360
 642         }
 643         else if (isJitStressScenario(scenario)) {
 644             timeout = 300
 645         }
 646         else if (isR2RBaselineScenario(scenario)) {
 647             timeout = 240
 648         }
 649         else if (isLongGc(scenario)) {
 650             timeout = 1440
 651         }
 652         else if (isJitDiff(scenario)) {
 653             timeout = 240
 654         }
 655         else if (isGcReliabilityFramework(scenario)) {
 656             timeout = 1440
 657         }
 658         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 659             timeout = 240
 660         }
 661     }
 662
 663     if (configuration == 'Debug') {
 664         // Debug runs can be very slow. Add an hour.
 665         timeout += 60
 666     }
 667
 668         if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 669                 // AltJit runs compile all methods twice.
 670                 timeout *= 2
 671         }
 672
 673     // If we've changed the timeout from the default, set it in the job.
 674
 675     if (timeout != 120) {
 676         Utilities.setJobTimeout(newJob, timeout)
 677     }
 678 }
 679
 680 def static getJobFolder(def scenario) {
 681     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 682         return 'jitstress'
 683     }
 684     if (scenario == 'illink') {
 685         return 'illink'
 686     }
 687     return ''
 688 }
 689
 690 def static getStressModeDisplayName(def scenario) {
 691     def displayStr = ''
 692     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 693         def prefixLength = 'COMPlus_'.length()
 694         if (k.length() >= prefixLength) {
 695             def modeName = k.substring(prefixLength, k.length())
 696             displayStr += ' ' + modeName + '=' + v
 697         }
 698     }
 699
 700     if (isCoreFxScenario(scenario)) {
 701         displayStr = ('CoreFx ' + displayStr).trim()
 702     }
 703
 704     return displayStr
 705 }
 706
 707 def static getR2RDisplayName(def scenario) {
 708     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 709     def displayStr = scenario
 710     def prefixLength = 'r2r_'.length()
 711     if (displayStr.length() >= prefixLength) {
 712         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 713     } else if (scenario == 'r2r') {
 714         displayStr = "R2R"
 715     }
 716     return displayStr
 717 }
 718
 719 //
 720 // Functions to create an environment script.
 721 //      envScriptCreate -- initialize the script (call first)
 722 //      envScriptFinalize -- finalize the script (call last)
 723 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 724 //      envScriptAppendExistingScript -- append an existing script to the generated script
 725 //
 726 // Each script returns a string of commands. Concatenate all the strings together before
 727 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 728 //
 729
 730 // Initialize the environment setting script.
 731 def static envScriptCreate(def os, def stepScriptLocation) {
 732     def stepScript = ''
 733     if (os == 'Windows_NT') {
 734         stepScript += "echo Creating TestEnv script\r\n"
 735         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 736
 737         // Create at least an empty script.
 738         stepScript += "echo. > ${stepScriptLocation}\r\n"
 739     }
 740     else {
 741         stepScript += "echo Creating environment setting script\n"
 742         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 743     }
 744
 745     return stepScript
 746 }
 747
 748 // Generates the string for setting stress mode variables.
 749 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 750     def stepScript = ''
 751     if (os == 'Windows_NT') {
 752         stressModeVars.each{ k, v ->
 753             // Write out what we are writing to the script file
 754             stepScript += "echo Setting ${k}=${v}\r\n"
 755             // Write out the set itself to the script file`
 756             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 757         }
 758     }
 759     else {
 760         stressModeVars.each{ k, v ->
 761             // Write out what we are writing to the script file
 762             stepScript += "echo Setting ${k}=${v}\n"
 763             // Write out the set itself to the script file`
 764             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 765         }
 766     }
 767
 768     return stepScript
 769 }
 770
 771 // Append an existing script to an environment script.
 772 // Returns string of commands to do this.
 773 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 774     assert (os == 'Windows_NT')
 775     def stepScript = ''
 776
 777     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 778     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 779
 780     return stepScript
 781 }
 782
 783 // Finalize an environment setting script.
 784 // Returns string of commands to do this.
 785 def static envScriptFinalize(def os, def stepScriptLocation) {
 786     def stepScript = ''
 787
 788     if (os == 'Windows_NT') {
 789         // Display the resulting script. This is useful when looking at the output log file.
 790         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 791         stepScript += "type ${stepScriptLocation}\r\n"
 792     }
 793     else {
 794         stepScript += "chmod +x ${stepScriptLocation}\n"
 795     }
 796
 797     return stepScript
 798 }
 799
 800 def static isNeedDocker(def architecture, def os, def isBuild) {
 801     if (isBuild) {
 802         if (architecture == 'x86' && os == 'Ubuntu') {
 803             return true
 804         }
 805         else if (architecture == 'arm') {
 806             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 807                 return true
 808             }
 809         }
 810     }
 811     else {
 812         if (architecture == 'x86' && os == 'Ubuntu') {
 813             return true
 814         }
 815     }
 816     return false
 817 }
 818
 819 def static getDockerImageName(def architecture, def os, def isBuild) {
 820     // We must change some docker private images to official later
 821     if (isBuild) {
 822         if (architecture == 'x86' && os == 'Ubuntu') {
 823             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 824         }
 825         else if (architecture == 'arm') {
 826             if (os == 'Ubuntu') {
 827                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 828             }
 829             else if (os == 'Ubuntu16.04') {
 830                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 831             }
 832             else if (os == 'Tizen') {
 833                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 834             }
 835         }
 836     }
 837     else {
 838         if (architecture == 'x86' && os == 'Ubuntu') {
 839             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 840         }
 841     }
 842     println("Unknown architecture to use docker: ${architecture} ${os}");
 843     assert false
 844 }
 845
 846 // Calculates the name of the build job based on some typical parameters.
 847 //
 848 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 849     // If the architecture is x64, do not add that info into the build name.
 850     // Need to change around some systems and other builds to pick up the right builds
 851     // to do that.
 852
 853     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 854     if (isBuildOnly) {
 855         suffix += '_bld'
 856     }
 857     def baseName = ''
 858     switch (architecture) {
 859         case 'x64':
 860             if (scenario == 'normal') {
 861                 // For now we leave x64 off of the name for compatibility with other jobs
 862                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 863             }
 864             else if (scenario == 'formatting') {
 865                 // we don't care about the configuration for the formatting job. It runs all configs
 866                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 867             }
 868             else {
 869                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 870             }
 871             break
 872         case 'arm64':
 873             if (os.toLowerCase() == "windows_nt") {
 874                 // These are cross builds
 875                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 876             }
 877             else {
 878                 // Defaults to a small page size set of machines.
 879                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 880             }
 881             break
 882         case 'arm':
 883             // These are cross builds
 884             if (os == 'Tizen') {
 885                 // ABI: softfp
 886                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 887             }
 888             else {
 889                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 890             }
 891             break
 892         case 'armlb':
 893             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 894             break
 895         case 'x86':
 896         case 'x86_arm_altjit':
 897         case 'x64_arm64_altjit':
 898             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 899             break
 900         default:
 901             println("Unknown architecture: ${architecture}");
 902             assert false
 903             break
 904     }
 905
 906     return baseName + suffix
 907 }
 908
 909 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 910
 911     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 912     if (os == 'Windows_NT') {
 913         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 914             if (!(branch in Constants.WindowsArm64Branches)) {
 915                 return
 916             }
 917         }
 918     }
 919
 920     // Check scenario.
 921     switch (scenario) {
 922         case 'innerloop':
 923             break
 924         case 'normal':
 925             switch (architecture) {
 926                 case 'x64':
 927                 case 'x86':
 928                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 929                         addPeriodicTriggerHelper(job, '@daily')
 930                     }
 931                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 932                         addGithubPushTriggerHelper(job)
 933                     }
 934                     break
 935                 case 'arm':
 936                 case 'armlb':
 937                 case 'x86_arm_altjit':
 938                 case 'x64_arm64_altjit':
 939                     addGithubPushTriggerHelper(job)
 940                     break
 941                 case 'arm64':
 942                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 943                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 944                     break
 945                 default:
 946                     println("Unknown architecture: ${architecture}");
 947                     assert false
 948                     break
 949             }
 950             break
 951         case 'r2r':
 952             assert !(os in bidailyCrossList)
 953             // r2r gets a push trigger for checked/release
 954             if (configuration == 'Checked' || configuration == 'Release') {
 955                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 956                 if (architecture == 'x64' && os != 'OSX10.12') {
 957                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 958                     if (isFlowJob || os == 'Windows_NT') {
 959                         addGithubPushTriggerHelper(job)
 960                     }
 961                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 962                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 963                     if (isFlowJob) {
 964                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 965                     }
 966                 }
 967                 // For x86, only add per-commit jobs for Windows
 968                 else if (architecture == 'x86') {
 969                     if (os == 'Windows_NT') {
 970                         addGithubPushTriggerHelper(job)
 971                     }
 972                 }
 973                 // arm64 r2r jobs should only run daily.
 974                 else if (architecture == 'arm64') {
 975                     if (os == 'Windows_NT') {
 976                         addPeriodicTriggerHelper(job, '@daily')
 977                     }
 978                 }
 979             }
 980             break
 981         case 'r2r_jitstress1':
 982         case 'r2r_jitstress2':
 983         case 'r2r_jitstressregs1':
 984         case 'r2r_jitstressregs2':
 985         case 'r2r_jitstressregs3':
 986         case 'r2r_jitstressregs4':
 987         case 'r2r_jitstressregs8':
 988         case 'r2r_jitstressregs0x10':
 989         case 'r2r_jitstressregs0x80':
 990         case 'r2r_jitstressregs0x1000':
 991         case 'r2r_jitminopts':
 992         case 'r2r_jitforcerelocs':
 993         case 'r2r_gcstress15':
 994             assert !(os in bidailyCrossList)
 995
 996             // GCStress=C is currently not supported on OS X
 997             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 998                 break
 999             }
1000
1001             // GC Stress 15 r2r gets a push trigger for checked/release
1002             if (configuration == 'Checked' || configuration == 'Release') {
1003                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1004                 if (architecture == 'x64') {
1005                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1006                     if (isFlowJob || os == 'Windows_NT') {
1007                         // Add a weekly periodic trigger
1008                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1009                     }
1010                 }
1011                 // For x86, only add per-commit jobs for Windows
1012                 else if (architecture == 'x86') {
1013                     if (os == 'Windows_NT') {
1014                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1015                     }
1016                 }
1017             }
1018             break
1019         case 'longgc':
1020             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1021             assert configuration == 'Release'
1022             assert architecture == 'x64'
1023             addPeriodicTriggerHelper(job, '@daily')
1024             // TODO: Add once external email sending is available again
1025             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1026             break
1027         case 'gcsimulator':
1028             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1029             assert configuration == 'Release'
1030             assert architecture == 'x64'
1031             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1032             // TODO: Add once external email sending is available again
1033             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1034             break
1035         case 'standalone_gc':
1036             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1037             assert (configuration == 'Release' || configuration == 'Checked')
1038             // TODO: Add once external email sending is available again
1039             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1040             addPeriodicTriggerHelper(job, '@daily')
1041             break
1042         case 'gc_reliability_framework':
1043             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1044             assert (configuration == 'Release' || configuration == 'Checked')
1045             // Only triggered by phrase.
1046             break
1047         case 'ilrt':
1048             assert !(os in bidailyCrossList)
1049             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1050             if (architecture == 'x64' && configuration == 'Release') {
1051                 // We don't expect to see a job generated except in these scenarios
1052                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1053                 if (isFlowJob || os == 'Windows_NT') {
1054                     addPeriodicTriggerHelper(job, '@daily')
1055                 }
1056             }
1057             break
1058         case 'jitdiff':
1059             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1060             assert configuration == 'Checked'
1061             assert (architecture == 'x64' || architecture == 'x86')
1062             addGithubPushTriggerHelper(job)
1063             break
1064         case 'formatting':
1065             assert (os == 'Windows_NT' || os == "Ubuntu")
1066             assert architecture == 'x64'
1067             addGithubPushTriggerHelper(job)
1068             break
1069         case 'jitstressregs1':
1070         case 'jitstressregs2':
1071         case 'jitstressregs3':
1072         case 'jitstressregs4':
1073         case 'jitstressregs8':
1074         case 'jitstressregs0x10':
1075         case 'jitstressregs0x80':
1076         case 'jitstressregs0x1000':
1077         case 'minopts':
1078         case 'forcerelocs':
1079         case 'jitstress1':
1080         case 'jitstress2':
1081         case 'jitstress2_jitstressregs1':
1082         case 'jitstress2_jitstressregs2':
1083         case 'jitstress2_jitstressregs3':
1084         case 'jitstress2_jitstressregs4':
1085         case 'jitstress2_jitstressregs8':
1086         case 'jitstress2_jitstressregs0x10':
1087         case 'jitstress2_jitstressregs0x80':
1088         case 'jitstress2_jitstressregs0x1000':
1089         case 'tailcallstress':
1090         case 'jitsse2only':
1091         case 'jitnosimd':
1092         case 'jitnox86hwintrinsic':
1093         case 'jitincompletehwintrinsic':
1094         case 'jitx86hwintrinsicnoavx':
1095         case 'jitx86hwintrinsicnoavx2':
1096         case 'jitx86hwintrinsicnosimd':
1097         case 'corefx_baseline':
1098         case 'corefx_minopts':
1099         case 'corefx_jitstress1':
1100         case 'corefx_jitstress2':
1101         case 'corefx_jitstressregs1':
1102         case 'corefx_jitstressregs2':
1103         case 'corefx_jitstressregs3':
1104         case 'corefx_jitstressregs4':
1105         case 'corefx_jitstressregs8':
1106         case 'corefx_jitstressregs0x10':
1107         case 'corefx_jitstressregs0x80':
1108         case 'corefx_jitstressregs0x1000':
1109         case 'zapdisable':
1110             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1111                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1112                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1113                     if (os == 'Windows_NT') {
1114                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1115                         addPeriodicTriggerHelper(job, '@weekly')
1116                     }
1117                 }
1118                 else {
1119                     addPeriodicTriggerHelper(job, '@daily')
1120                 }
1121             }
1122             break
1123         case 'heapverify1':
1124         case 'gcstress0x3':
1125             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1126                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1127                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1128                     if (os == 'Windows_NT') {
1129                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1130                         addPeriodicTriggerHelper(job, '@weekly')
1131                     }
1132                     // TODO: Add once external email sending is available again
1133                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1134                 }
1135                 else {
1136                     addPeriodicTriggerHelper(job, '@weekly')
1137                 }
1138             }
1139             break
1140         case 'gcstress0xc':
1141         case 'gcstress0xc_zapdisable':
1142         case 'gcstress0xc_zapdisable_jitstress2':
1143         case 'gcstress0xc_zapdisable_heapverify1':
1144         case 'gcstress0xc_jitstress1':
1145         case 'gcstress0xc_jitstress2':
1146         case 'gcstress0xc_minopts_heapverify1':
1147             // GCStress=C is currently not supported on OS X
1148             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1149                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1150                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1151                     if (os == 'Windows_NT') {
1152                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1153                         addPeriodicTriggerHelper(job, '@weekly')
1154                     }
1155                     // TODO: Add once external email sending is available again
1156                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1157                 }
1158                 else {
1159                     addPeriodicTriggerHelper(job, '@weekly')
1160                 }
1161             }
1162             break
1163
1164         case 'illink':
1165             // Testing on other operating systems TBD
1166             assert (os == 'Windows_NT' || os == 'Ubuntu')
1167             if (architecture == 'x64' || architecture == 'x86') {
1168                 if (configuration == 'Checked') {
1169                     addPeriodicTriggerHelper(job, '@daily')
1170                 }
1171             }
1172             break
1173
1174         case 'tieredcompilation':
1175         case 'corefx_tieredcompilation':
1176             // No periodic jobs just yet, still testing
1177             break
1178
1179         default:
1180             println("Unknown scenario: ${scenario}");
1181             assert false
1182             break
1183     }
1184     return
1185 }
1186
1187 // **************************
1188 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1189 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1190 // builds will do a couple extra steps.
1191 // **************************
1192
1193 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1194 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1195 // it's just used for internal builds
1196 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1197 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1198     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1199
1200     if (isWindowsBuildOnlyJob) {
1201         return
1202     }
1203
1204     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1205     // Non pull request builds.
1206     if (!isPR) {
1207         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1208         return
1209     }
1210
1211      def arm64Users = [
1212         'AndyAyersMS',
1213         'briansull',
1214         'BruceForstall',
1215         'CarolEidt',
1216         'cmckinsey',
1217         'erozenfeld',
1218         'janvorli',
1219         'jashook',
1220         'JosephTremoulet',
1221         'pgodeq',
1222         'pgavlin',
1223         'rartemev',
1224         'russellhadley',
1225         'RussKeldorph',
1226         'sandreenko',
1227         'sdmaclea',
1228         'swaroop-sridhar',
1229         'jkotas',
1230         'markwilkie',
1231         'weshaggard'
1232     ]
1233
1234     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1235     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1236     def osGroup = getOSGroup(os)
1237     switch (architecture) {
1238         case 'x64': // editor brace matching: {
1239             if (scenario == 'formatting') {
1240                 assert configuration == 'Checked'
1241                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1242                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1243                 }
1244
1245                 break
1246             }
1247
1248             switch (os) {
1249                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1250                 case 'Debian8.4':
1251                 case 'RHEL7.2':
1252                     if (scenario == 'innerloop') {
1253                         assert !isFlowJob
1254                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1255                     }
1256                     else if (scenario == 'normal') {
1257                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1258                     }
1259                     break
1260
1261                 case 'Ubuntu16.04':
1262                     assert !isFlowJob
1263                     assert scenario != 'innerloop'
1264                     // Distinguish with the other architectures (arm and x86)
1265                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1266                     break
1267
1268                 case 'Fedora24':
1269                 case 'Ubuntu16.10':
1270                     assert !isFlowJob
1271                     assert scenario != 'innerloop'
1272                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1273                     break
1274
1275                 case 'Ubuntu':
1276                     if (scenario == 'illink') {
1277                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                         break
1279                     }
1280                     // fall through
1281
1282                 case 'OSX10.12':
1283                     // Triggers on the non-flow jobs aren't necessary here
1284                     // Corefx testing uses non-flow jobs.
1285                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1286                         break
1287                     }
1288                     switch (scenario) {
1289                         case 'innerloop':
1290                             // PR Triggered jobs. These jobs will run pri0 tests.
1291                             if (configuration == 'Checked') {
1292                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1293                             }
1294                             break
1295
1296                         case 'normal':
1297                             // OSX uses checked for default PR tests
1298                             if (configuration == 'Checked') {
1299                                 // Default trigger
1300                                 assert !job.name.contains("centos")
1301                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1302                             }
1303                             break
1304
1305                         case 'jitdiff':
1306                             if (configuration == 'Checked') {
1307                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1308                             }
1309                             break
1310
1311                         case 'ilrt':
1312                             if (configuration == 'Release') {
1313                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1314                             }
1315                             break
1316
1317                         case 'longgc':
1318                             if (configuration == 'Release') {
1319                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1320                             }
1321                             break
1322
1323                         case 'gcsimulator':
1324                             if (configuration == 'Release') {
1325                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1326                             }
1327                             break
1328
1329                         case 'standalone_gc':
1330                             if (configuration == 'Release' || configuration == 'Checked') {
1331                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1332                             }
1333                             break
1334
1335                         case 'gc_reliability_framework':
1336                             if (configuration == 'Release' || configuration == 'Checked') {
1337                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1338                             }
1339                             break
1340
1341                         default:
1342                             if (isJitStressScenario(scenario)) {
1343                                 def displayStr = getStressModeDisplayName(scenario)
1344                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1345                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1346                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1347                             }
1348                             else if (isR2RScenario(scenario)) {
1349                                 if (configuration == 'Release' || configuration == 'Checked') {
1350                                     def displayStr = getR2RDisplayName(scenario)
1351                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1352                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1353                                 }
1354                             }
1355                             else {
1356                                 println("Unknown scenario: ${scenario}");
1357                                 assert false
1358                             }
1359                             break
1360
1361                     }
1362                     break
1363
1364                 case 'CentOS7.1':
1365                     switch (scenario) {
1366                         case 'innerloop':
1367                             // CentOS uses checked for default PR tests while debug is build only
1368                             if (configuration == 'Debug') {
1369                                 // Default trigger
1370                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1371                             }
1372
1373                             // Make sure this is a flow job to get build and test.
1374                             if (configuration == 'Checked' && isFlowJob) {
1375                                 assert job.name.contains("flow")
1376                                 // Default trigger
1377                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1378                             }
1379                             break
1380
1381                         case 'normal':
1382                             // Make sure this is a flow job to get build and test.
1383                             if (configuration == 'Checked' && isFlowJob) {
1384                                 assert job.name.contains("flow")
1385                                 // Default trigger
1386                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1387                             }
1388                             break
1389
1390                         default:
1391                             if (isR2RScenario(scenario)) {
1392                                 if (configuration == 'Release' || configuration == 'Checked') {
1393                                     def displayStr = getR2RDisplayName(scenario)
1394                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1395                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1396                                 }
1397                             }
1398                             break
1399
1400                     }
1401                     break
1402
1403                 case 'Windows_NT':
1404                     switch (scenario) {
1405                         case 'innerloop':
1406                             // Default trigger
1407                             if (configuration == 'Checked' || configuration == 'Release') {
1408                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1409                             }
1410                             break
1411
1412                         case 'normal':
1413                             if (configuration == 'Checked') {
1414                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1415                             }
1416                             break
1417
1418                         case 'jitdiff':
1419                             if (configuration == 'Checked') {
1420                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1421                             }
1422                             break
1423
1424                         case 'ilrt':
1425                             if (configuration == 'Release') {
1426                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1427                             }
1428                             break
1429
1430                         case 'longgc':
1431                             if (configuration == 'Release') {
1432                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1433                             }
1434                             break
1435
1436                         case 'gcsimulator':
1437                             if (configuration == 'Release') {
1438                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1439                             }
1440                             break
1441
1442                         case 'standalone_gc':
1443                             if (configuration == 'Release' || configuration == 'Checked') {
1444                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1445                             }
1446                             break
1447
1448                         case 'gc_reliability_framework':
1449                             if (configuration == 'Release' || configuration == 'Checked') {
1450                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1451                             }
1452                             break
1453
1454                         case 'illink':
1455                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1456                             break
1457
1458                         default:
1459                             if (isJitStressScenario(scenario)) {
1460                                 def displayStr = getStressModeDisplayName(scenario)
1461                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1462                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1463                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1464                             }
1465                             else if (isR2RScenario(scenario)) {
1466                                 if (configuration == 'Release' || configuration == 'Checked') {
1467                                     def displayStr = getR2RDisplayName(scenario)
1468                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1469                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1470                                 }
1471                             }
1472                             else {
1473                                 println("Unknown scenario: ${scenario}");
1474                                 assert false
1475                             }
1476                             break
1477
1478                     }
1479                     break
1480
1481                 default:
1482                     println("Unknown os: ${os}");
1483                     assert false
1484                     break
1485
1486             }
1487
1488             break
1489
1490         // editor brace matching: }
1491         case 'armlb':
1492         case 'arm': // editor brace matching: {
1493             switch (os) {
1494                 case 'Ubuntu':
1495                 case 'Ubuntu16.04':
1496                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1497                         break
1498                     }
1499
1500                     assert scenario != 'innerloop'
1501                     job.with {
1502                         publishers {
1503                             azureVMAgentPostBuildAction {
1504                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1505                             }
1506                         }
1507                     }
1508                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1509                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1510                     break
1511
1512                 case 'Tizen':
1513                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1514                         break
1515                     }
1516
1517                     architecture = 'armel'
1518                     job.with {
1519                         publishers {
1520                             azureVMAgentPostBuildAction {
1521                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1522                             }
1523                         }
1524                     }
1525
1526                     if (scenario == 'innerloop') {
1527                         if (configuration == 'Checked') {
1528                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1529                         }
1530                     }
1531                     else {
1532                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1533                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1534                     }
1535                     break
1536
1537                 case 'Windows_NT':
1538                     if (architecture == "armlb") {
1539                         // Disable armlb windows jobs
1540                         break
1541                     }
1542
1543                     // Triggers on the non-flow jobs aren't necessary here
1544                     if (!isFlowJob) {
1545                         break
1546                     }
1547
1548                     // Set up a private trigger
1549                     def contextString = "${os} ${architecture} Cross ${configuration}"
1550                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1551                     if (scenario == 'innerloop') {
1552                         contextString += " Innerloop"
1553                         triggerString += "\\W+Innerloop"
1554                     }
1555                     else {
1556                         contextString += " ${scenario}"
1557                         triggerString += "\\W+${scenario}"
1558                     }
1559
1560                     if (configuration == 'Debug') {
1561                         contextString += " Build"
1562                         triggerString += "\\W+Build"
1563                     } else {
1564                         contextString += " Build and Test"
1565                         triggerString += "\\W+Build and Test"
1566                     }
1567
1568                     triggerString += ".*"
1569
1570                     switch (scenario) {
1571                         case 'innerloop':
1572                             // Only Checked is an innerloop trigger.
1573                             if (configuration == 'Checked')
1574                             {
1575                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1576                             }
1577                             break
1578                         case 'normal':
1579                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1580                             break
1581                         default:
1582                             // Stress jobs will use this code path.
1583                             if (isArmWindowsScenario(scenario)) {
1584                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1585                             }
1586                             break
1587                     }
1588                     break
1589                 default:
1590                     println("NYI os: ${os}");
1591                     assert false
1592                     break
1593             }
1594             break
1595         // editor brace matching: }
1596         case 'arm64': // editor brace matching: {
1597             // Set up a private trigger
1598             def contextString = "${os} ${architecture} Cross ${configuration}"
1599             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1600
1601             if (scenario == 'innerloop') {
1602                 contextString += " Innerloop"
1603                 triggerString += "\\W+Innerloop"
1604             }
1605             else {
1606                 contextString += " ${scenario}"
1607                 triggerString += "\\W+${scenario}"
1608             }
1609
1610             if (configuration == 'Debug') {
1611                 contextString += " Build"
1612                 triggerString += "\\W+Build"
1613             } else {
1614                 contextString += " Build and Test"
1615                 triggerString += "\\W+Build and Test"
1616             }
1617
1618             triggerString += ".*"
1619
1620             switch (os) {
1621                 case 'Ubuntu':
1622                 case 'Ubuntu16.04':
1623                     switch (scenario) {
1624                         case 'innerloop':
1625                             if (configuration == 'Debug' && !isFlowJob) {
1626                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1627                             }
1628
1629                             break
1630                         case 'normal':
1631                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1632                             break
1633                         default:
1634                             if (isR2RScenario(scenario)) {
1635                                 if (configuration == 'Checked' || configuration == 'Release') {
1636                                     def displayStr = getR2RDisplayName(scenario)
1637                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1638                                 }
1639                             }
1640                             break
1641                     }
1642                     break
1643
1644                 case 'Windows_NT':
1645                     // Triggers on the non-flow jobs aren't necessary here
1646                     if (!isFlowJob) {
1647                         break
1648                     }
1649
1650                     assert isArmWindowsScenario(scenario)
1651                     switch (scenario) {
1652                         case 'innerloop':
1653                             if (configuration == 'Checked') {
1654                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1655                             }
1656
1657                             break
1658                         case 'normal':
1659                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1660                             break
1661                         default:
1662                             // Stress jobs will use this code path.
1663                             if (isArmWindowsScenario(scenario)) {
1664                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1665                             }
1666                             break
1667                     }
1668                     break
1669                 default:
1670                     println("NYI os: ${os}");
1671                     assert false
1672                     break
1673             }
1674             break
1675
1676         // editor brace matching: }
1677         case 'x86': // editor brace matching: {
1678             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1679             if (os == 'Ubuntu') {
1680                 // Triggers on the non-flow jobs aren't necessary here
1681                 if (!isFlowJob) {
1682                     break
1683                 }
1684
1685                 // on-demand only for ubuntu x86
1686                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1687                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1688                 break
1689
1690             }
1691             switch (scenario) {
1692                 case 'innerloop':
1693                     if (configuration == 'Checked' || configuration == 'Release') {
1694                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1695                     }
1696                     break
1697
1698                 case 'normal':
1699                     if (configuration == 'Checked') {
1700                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1701                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1702                     }
1703                     break
1704
1705                 case 'ilrt':
1706                     if (configuration == 'Release') {
1707                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1708                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1709                     }
1710                     break
1711
1712                 case 'longgc':
1713                     if (configuration == 'Release') {
1714                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1715                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1716                     }
1717                     break
1718
1719                 case 'gcsimulator':
1720                     if (configuration == 'Release') {
1721                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1722                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1723                     }
1724                     break
1725
1726                 case 'standalone_gc':
1727                     if (configuration == 'Release' || configuration == 'Checked') {
1728                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1729                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1730                     }
1731                     break
1732
1733                 case 'illink':
1734                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1735                     break
1736
1737                 default:
1738                     if (isJitStressScenario(scenario)) {
1739                         def displayStr = getStressModeDisplayName(scenario)
1740                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1741                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1742                     }
1743                     else if (isR2RScenario(scenario)) {
1744                         if (configuration == 'Release' || configuration == 'Checked') {
1745                             def displayStr = getR2RDisplayName(scenario)
1746                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1747                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1748                         }
1749                     }
1750                     else {
1751                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1752                         assert false
1753                     }
1754                     break
1755
1756             }
1757             break
1758
1759          // editor brace matching: }
1760         case 'x64_arm64_altjit':
1761         case 'x86_arm_altjit': // editor brace matching: {
1762             assert (os == 'Windows_NT')
1763             switch (scenario) {
1764                 case 'normal':
1765                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1766                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1767                     break
1768                 default:
1769                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1770                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1771                     break
1772             }
1773             break
1774
1775         // editor brace matching: }
1776         default:
1777             println("Unknown architecture: ${architecture}");
1778             assert false
1779             break
1780     }
1781 }
1782
1783 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1784     def buildCommands = [];
1785     def osGroup = getOSGroup(os)
1786     def lowerConfiguration = configuration.toLowerCase()
1787
1788     def priority = '1'
1789     if (scenario == 'innerloop') {
1790         priority = '0'
1791     }
1792
1793     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1794
1795     def enableCorefxTesting = isCoreFxScenario(scenario)
1796
1797     // Calculate the build steps, archival, and xunit results
1798     switch (os) {
1799         case 'Windows_NT': // editor brace matching: {
1800             switch (architecture) {
1801                 case 'x64':
1802                 case 'x86':
1803                 case 'x86_arm_altjit':
1804                 case 'x64_arm64_altjit':
1805                     def arch = architecture
1806                     def buildOpts = ''
1807                     if (architecture == 'x86_arm_altjit') {
1808                         arch = 'x86'
1809                     }
1810                     else if (architecture == 'x64_arm64_altjit') {
1811                         arch = 'x64'
1812                     }
1813
1814                     if (scenario == 'formatting') {
1815                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1816                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1817                         break
1818                     }
1819
1820                     if (scenario == 'illink') {
1821                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1822                     }
1823
1824                     // If it is a release build for windows, ensure PGO is used, else fail the build
1825                     if ((lowerConfiguration == 'release') &&
1826                         (scenario in Constants.basicScenarios) &&
1827                         (architecture != 'x86_arm_altjit') &&
1828                         (architecture != 'x64_arm64_altjit')) {
1829
1830                         buildOpts += ' -enforcepgo'
1831                     }
1832
1833                     if (enableCorefxTesting) {
1834                         buildOpts += ' skiptests';
1835                     } else {
1836                         buildOpts += " -priority=${priority}"
1837                     }
1838
1839                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1840                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1841                     // 35 characters long.
1842
1843                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1844
1845                     if (!isBuildOnly) {
1846                         def runtestArguments = ''
1847                         def testOpts = 'collectdumps'
1848
1849                         if (isR2RScenario(scenario)) {
1850
1851                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1852                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1853                             // to cause the tests to be crossgen'ed.
1854
1855                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1856                                 testOpts += ' crossgenaltjit protononjit.dll'
1857                             } else {
1858                                 testOpts += ' crossgen'
1859                             }
1860
1861                             testOpts += ' runcrossgentests'
1862                         }
1863                         else if (scenario == 'jitdiff') {
1864                             testOpts += ' jitdisasm crossgen'
1865                         }
1866                         else if (scenario == 'ilrt') {
1867                             testOpts += ' ilasmroundtrip'
1868                         }
1869                         else if (isLongGc(scenario)) {
1870                             testOpts += " ${scenario} sequential"
1871                         }
1872                         else if (scenario == 'standalone_gc') {
1873                             testOpts += ' gcname clrgc.dll'
1874                         }
1875                         else if (scenario == 'illink') {
1876                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1877                         }
1878
1879                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1880                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1881                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1882                         // which is read by the xunit harness.)
1883                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1884                         {
1885                             def timeout = 1800000
1886                             testOpts += " timeout ${timeout}"
1887                         }
1888
1889                         // If we are running a stress mode, we should write out the set of key
1890                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1891
1892                         def envScriptPath = ''
1893                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1894                             def buildCommandsStr = ''
1895                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1896                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1897
1898                             if (isJitStressScenario(scenario)) {
1899                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1900                             }
1901                             else if (isR2RStressScenario(scenario)) {
1902                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1903                             }
1904
1905                             if (architecture == 'x86_arm_altjit') {
1906                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1907                             }
1908                             else if (architecture == 'x64_arm64_altjit') {
1909                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1910                             }
1911
1912                             envScriptFinalize(os, envScriptPath)
1913
1914                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1915                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1916                             // into a single script.
1917                             buildCommands += buildCommandsStr
1918                         }
1919                         else if (architecture == 'x86_arm_altjit') {
1920                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1921                         }
1922                         else if (architecture == 'x64_arm64_altjit') {
1923                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1924                         }
1925                         if (envScriptPath != '') {
1926                             testOpts += " TestEnv ${envScriptPath}"
1927                         }
1928
1929                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1930
1931                         if (enableCorefxTesting) {
1932                             def workspaceRelativeFxRoot = "_/fx"
1933                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1934
1935                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1936
1937                             // Archive and process (only) the test results
1938                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1939                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1940
1941                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1942                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1943                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1944                         }
1945                         else if (isGcReliabilityFramework(scenario)) {
1946                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1947                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1948                         }
1949                         else {
1950                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1951                         }
1952                     }
1953
1954                     if (!enableCorefxTesting) {
1955                         // Run the rest of the build
1956                         // Build the mscorlib for the other OS's
1957                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1958                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1959
1960                         if (arch == "x64") {
1961                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1962                         }
1963
1964                         // Zip up the tests directory so that we don't use so much space/time copying
1965                         // 10s of thousands of files around.
1966                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1967
1968                         if (!isJitStressScenario(scenario)) {
1969                             // For Windows, pull full test results and test drops for x86/x64.
1970                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1971                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1972                         }
1973
1974                         if (scenario == 'jitdiff') {
1975                             // retrieve jit-dasm output for base commit, and run jit-diff
1976                             if (!isBuildOnly) {
1977                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1978                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1979                             }
1980                         }
1981
1982                         if (!isBuildOnly) {
1983                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1984                         }
1985                     }
1986                     break
1987                 case 'armlb':
1988                 case 'arm':
1989                     assert isArmWindowsScenario(scenario)
1990
1991                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1992                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1993
1994                     def buildArchitecture = 'arm'
1995
1996                     def buildOpts = ''
1997
1998                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1999                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2000                     if (architecture == 'armlb') {
2001                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2002                     }
2003
2004                     if (enableCorefxTesting) {
2005                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2006                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2007                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2008                         //
2009                         // buildOpts += ' skiptests';
2010                         buildOpts += " -priority=0"
2011                     } else {
2012                         buildOpts += " -priority=${priority}"
2013                     }
2014
2015                     // This is now a build only job. Do not run tests. Use the flow job.
2016                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2017
2018                     if (enableCorefxTesting) {
2019                         assert isBuildOnly
2020                         assert architecture == 'arm'
2021
2022                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2023                         // to determine the correct matching corefx version git commit hash.
2024                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2025
2026                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2027                         def envScriptPath = ''
2028                         def buildCommandsStr = ''
2029                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2030                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2031                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2032                         envScriptFinalize(os, envScriptPath)
2033                         buildCommands += buildCommandsStr
2034
2035                         def workspaceRelativeFxRootLinux = "_/fx"
2036                         def workspaceRelativeFxRootWin = "_\\fx"
2037                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2038
2039                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2040
2041                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2042                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2043                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2044
2045                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2046                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2047                     } else {
2048                         // Zip up the tests directory so that we don't use so much space/time copying
2049                         // 10s of thousands of files around.
2050                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2051
2052                         // Add archival.
2053                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2054                     }
2055                     break
2056                 case 'arm64':
2057                     assert isArmWindowsScenario(scenario)
2058
2059                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2060                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2061
2062                     // This is now a build only job. Do not run tests. Use the flow job.
2063                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2064
2065                     // Zip up the tests directory so that we don't use so much space/time copying
2066                     // 10s of thousands of files around.
2067                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2068
2069                     // Add archival.
2070                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2071                     break
2072                 default:
2073                     println("Unknown architecture: ${architecture}");
2074                     assert false
2075                     break
2076             }
2077             break
2078         // editor brace matching: }
2079         case 'Ubuntu':
2080         case 'Ubuntu16.04':
2081         case 'Ubuntu16.10':
2082         case 'Debian8.4':
2083         case 'OSX10.12':
2084         case 'CentOS7.1':
2085         case 'RHEL7.2':
2086         case 'Tizen':
2087         case 'Fedora24': // editor brace matching: {
2088             switch (architecture) {
2089                 case 'x64':
2090                 case 'x86':
2091                     if (architecture == 'x86' && os == 'Ubuntu') {
2092                         // build and PAL test
2093                         def dockerImage = getDockerImageName(architecture, os, true)
2094                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2095                         dockerImage = getDockerImageName(architecture, os, false)
2096                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2097                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2098                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2099                         break
2100                     }
2101
2102                     if (scenario == 'formatting') {
2103                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2104                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2105                         break
2106                     }
2107
2108                     if (scenario == 'illink') {
2109                         assert(os == 'Ubuntu')
2110                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2111                     }
2112
2113                     if (!enableCorefxTesting) {
2114                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2115                         // only on supported OS platforms.
2116                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2117                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2118
2119                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2120                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2121
2122                         // Basic archiving of the build
2123                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2124                         // And pal tests
2125                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2126                     }
2127                     else {
2128                         // Corefx stress testing
2129                         assert os == 'Ubuntu'
2130                         assert architecture == 'x64'
2131                         assert lowerConfiguration == 'checked'
2132                         assert isJitStressScenario(scenario)
2133
2134                         // Build coreclr
2135                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2136
2137                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2138
2139                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2140                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2141                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2142                         buildCommands += envScriptCmds
2143
2144                         // Build and text corefx
2145                         def workspaceRelativeFxRoot = "_/fx"
2146                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2147
2148                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2149
2150                         // Archive and process (only) the test results
2151                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2152                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2153                     }
2154                     break
2155                 case 'arm64':
2156                     if (!enableCorefxTesting) {
2157                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2158
2159                         // HACK -- Arm64 does not have corefx jobs yet.
2160                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2161                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2162                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2163                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2164
2165                         // Basic archiving of the build
2166                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2167                     }
2168                     break
2169                 case 'arm':
2170                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2171                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2172
2173                     // default values for Ubuntu
2174                     def arm_abi="arm"
2175                     def linuxCodeName="trusty"
2176                     if (os == 'Ubuntu16.04') {
2177                         linuxCodeName="xenial"
2178                     }
2179                     else if (os == 'Tizen') {
2180                         arm_abi="armel"
2181                         linuxCodeName="tizen"
2182                     }
2183
2184                     // Unzip the Windows test binaries first. Exit with 0
2185                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2186
2187                     // Unpack the corefx binaries
2188                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2189                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2190                     if (os != 'Tizen') {
2191                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2192                     }
2193                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2194                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2195                     if (os == 'Ubuntu') {
2196                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2197                     }
2198
2199                     // Call the ARM CI script to cross build and test using docker
2200                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2201                     --mode=docker \\
2202                     --${arm_abi} \\
2203                     --linuxCodeName=${linuxCodeName} \\
2204                     --buildConfig=${lowerConfiguration} \\
2205                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2206                     --coreFxBinDir=./bin/CoreFxBinDir \\
2207                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2208
2209                     // Basic archiving of the build, no pal tests
2210                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2211                     break
2212                 default:
2213                     println("Unknown architecture: ${architecture}");
2214                     assert false
2215                     break
2216             }
2217             break
2218         // editor brace matching: }
2219         default:
2220             println("Unknown os: ${os}");
2221             assert false
2222             break
2223     } // os
2224
2225     return buildCommands
2226 }
2227
2228 Constants.allScenarios.each { scenario ->
2229     [true, false].each { isPR ->
2230         Constants.architectureList.each { architecture ->
2231             Constants.configurationList.each { configuration ->
2232                 Constants.osList.each { os ->
2233                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2234                     // and reset the os to Windows_NT
2235                     def isBuildOnly = false
2236                     if (os == 'Windows_NT_BuildOnly') {
2237                         isBuildOnly = true
2238                         os = 'Windows_NT'
2239                     }
2240
2241                     // Tizen is only supported for arm architecture
2242                     if (os == 'Tizen' && architecture != 'arm') {
2243                         return
2244                     }
2245
2246                     // Skip totally unimplemented (in CI) configurations.
2247                     switch (architecture) {
2248                         case 'arm64':
2249                             if (os == 'Ubuntu16.04') {
2250                                 os = 'Ubuntu'
2251                             }
2252
2253                             // Windows and Ubuntu only
2254                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2255                                 return
2256                             }
2257                             break
2258                         case 'arm':
2259                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2260                                 return
2261                             }
2262                             break
2263                         case 'armlb':
2264                             // Do not create armlb jobs
2265                             return
2266                             break
2267                         case 'x86':
2268                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2269                                 return
2270                             }
2271                             break
2272                         case 'x86_arm_altjit':
2273                         case 'x64_arm64_altjit':
2274                             if (os != 'Windows_NT') {
2275                                 return
2276                             }
2277                             break
2278                         case 'x64':
2279                             // Everything implemented
2280                             break
2281                         default:
2282                             println("Unknown architecture: ${architecture}")
2283                             assert false
2284                             break
2285                     }
2286
2287                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2288                     // with checked builds)
2289                     if (isJitStressScenario(scenario)) {
2290                         if (configuration != 'Checked') {
2291                             return
2292                         }
2293
2294                         // Since these are just execution time differences,
2295                         // skip platforms that don't execute the tests here (Windows_NT only)
2296                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2297                         if (!isEnabledOS) {
2298                             return
2299                         }
2300
2301                         switch (architecture) {
2302                             case 'x64':
2303                             case 'x86':
2304                             case 'x86_arm_altjit':
2305                             case 'x64_arm64_altjit':
2306                                 // x86 ubuntu: default only
2307                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2308                                     return
2309                                 }
2310                                 if (isBuildOnly) {
2311                                     return
2312                                 }
2313                                 break
2314
2315                             case 'arm':
2316                                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2317                                 if (!isBuildOnly || !isCoreFxScenario(scenario)) {
2318                                     return
2319                                 }
2320                                 break
2321
2322                             default:
2323                                 // arm64, armlb: stress is handled through flow jobs.
2324                                 return
2325                         }
2326                     }
2327                     else if (isR2RScenario(scenario)) {
2328                         if (os != 'Windows_NT') {
2329                             return
2330                         }
2331                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2332                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2333                             return
2334                         }
2335                     }
2336                     else {
2337                         // Skip scenarios
2338                         switch (scenario) {
2339                             case 'ilrt':
2340                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2341                                 // the default scenario build
2342                                 if (os != 'Windows_NT') {
2343                                     return
2344                                 }
2345                                 // Only x64 for now
2346                                 if (architecture != 'x64') {
2347                                     return
2348                                 }
2349                                 // Release only
2350                                 if (configuration != 'Release') {
2351                                     return
2352                                 }
2353                                 break
2354                             case 'jitdiff':
2355                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2356                                     return
2357                                 }
2358                                 if (architecture != 'x64') {
2359                                     return
2360                                 }
2361                                 if (configuration != 'Checked') {
2362                                     return
2363                                 }
2364                                 break
2365                             case 'longgc':
2366                             case 'gcsimulator':
2367                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2368                                     return
2369                                 }
2370                                 if (architecture != 'x64') {
2371                                     return
2372                                 }
2373                                 if (configuration != 'Release') {
2374                                     return
2375                                 }
2376                                 break
2377                             case 'gc_reliability_framework':
2378                             case 'standalone_gc':
2379                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2380                                     return
2381                                 }
2382
2383                                 if (architecture != 'x64') {
2384                                     return
2385                                 }
2386
2387                                 if (configuration != 'Release' && configuration != 'Checked') {
2388                                     return
2389                                 }
2390                                 break
2391                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2392                             case 'formatting':
2393                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2394                                     return
2395                                 }
2396                                 if (architecture != 'x64') {
2397                                     return
2398                                 }
2399                                 if (configuration != 'Checked') {
2400                                     return
2401                                 }
2402                                 if (isBuildOnly) {
2403                                     return
2404                                 }
2405                                 break
2406                             case 'illink':
2407                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2408                                     return
2409                                 }
2410                                 if (architecture != 'x64' && architecture != 'x86') {
2411                                     return
2412                                 }
2413                                 if (isBuildOnly) {
2414                                     return
2415                                 }
2416                                 break
2417                             case 'normal':
2418                                 // Nothing skipped
2419                                 break
2420                             case 'innerloop':
2421                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2422                                     return
2423                                 }
2424                                 break
2425                             default:
2426                                 println("Unknown scenario: ${scenario}")
2427                                 assert false
2428                                 break
2429                         }
2430                     }
2431
2432                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2433                     // runtime behavior, not compile-time behavior, are not interesting.
2434                     switch (architecture) {
2435                         case 'x86_arm_altjit':
2436                         case 'x64_arm64_altjit':
2437                             if (isGCStressRelatedTesting(scenario)) {
2438                                 return
2439                             }
2440                             break
2441                         default:
2442                             break
2443                     }
2444
2445                     // Calculate names
2446                     def lowerConfiguration = configuration.toLowerCase()
2447                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2448                     def folderName = getJobFolder(scenario)
2449
2450                     // Create the new job
2451                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2452                     addToViews(newJob, isPR, architecture, os)
2453
2454                     def machineAffinityOptions = null
2455
2456                     if (os != 'Windows_NT') {
2457                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2458                     }
2459                     else {
2460                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2461                     }
2462
2463                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2464
2465                     // Add all the standard options
2466                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2467                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2468
2469                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2470                     def osGroup = getOSGroup(os)
2471
2472                     newJob.with {
2473                         steps {
2474                             if (os == 'Windows_NT') {
2475                                 buildCommands.each { buildCommand ->
2476                                     batchFile(buildCommand)
2477                                 }
2478                             }
2479                             else {
2480                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2481                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2482                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2483                                     // Define the Windows Tests and Corefx build job names
2484                                     def WindowsTestsName = projectFolder + '/' +
2485                                                            Utilities.getFullJobName(project,
2486                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2487                                                                                     false)
2488                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2489                                                        Utilities.getFolderName(branch)
2490
2491                                     // Copy the Windows test binaries and the Corefx build binaries
2492                                     copyArtifacts(WindowsTestsName) {
2493                                         includePatterns('bin/tests/tests.zip')
2494                                         buildSelector {
2495                                             latestSuccessful(true)
2496                                         }
2497                                     }
2498
2499                                     def arm_abi = 'arm'
2500                                     def corefx_os = 'linux'
2501                                     if (os == 'Tizen') {
2502                                         arm_abi = 'armel'
2503                                         corefx_os = 'tizen'
2504                                     }
2505
2506                                     // Let's use release CoreFX to test checked CoreCLR,
2507                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2508                                     def corefx_lowerConfiguration = lowerConfiguration
2509                                     if ( lowerConfiguration == 'checked' ) {
2510                                         corefx_lowerConfiguration='release'
2511                                     }
2512
2513                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2514                                         includePatterns('bin/build.tar.gz')
2515                                         buildSelector {
2516                                             latestSuccessful(true)
2517                                         }
2518                                     }
2519                                 }
2520
2521                                 buildCommands.each { buildCommand ->
2522                                     shell(buildCommand)
2523                                 }
2524                             }
2525                         }
2526                     } // newJob.with
2527
2528                 } // os
2529             } // configuration
2530         } // architecture
2531     } // isPR
2532 } // scenario
2533
2534
2535 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2536 Constants.allScenarios.each { scenario ->
2537     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2538
2539     [true, false].each { isPR ->
2540         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2541             Constants.crossList.each { os ->
2542                 if (architecture == 'arm64') {
2543                     if (os != "Ubuntu" && os != "Windows_NT") {
2544                         return
2545                     }
2546                 } else if (architecture == 'arm') {
2547                     if (os != 'Windows_NT') {
2548                         return
2549                     }
2550                 } else if (architecture == 'armlb') {
2551                   // Do not create armlb windows jobs.
2552                   return
2553                 } else if (architecture == 'x86') {
2554                     if (os != "Ubuntu") {
2555                         return
2556                     }
2557                 }
2558
2559                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2560
2561                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2562                     return
2563                 }
2564
2565                 Constants.configurationList.each { configuration ->
2566
2567                     // First, filter based on OS.
2568
2569                     if (os == 'Windows_NT') {
2570                         if (!isArmWindowsScenario(scenario)) {
2571                             return
2572                         }
2573                     }
2574                     else {
2575                         // Non-Windows
2576                         if (architecture == 'arm64') {
2577                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2578                                 return
2579                             }
2580                         }
2581                         else if (architecture == 'x86') {
2582                             // Linux/x86 only want innerloop and default test
2583                             if (!isNormalOrInnerloop) {
2584                                 return
2585                             }
2586                         }
2587                     }
2588
2589                     // For CentOS, we only want Checked/Release builds.
2590                     if (os == 'CentOS7.1') {
2591                         if (configuration != 'Checked' && configuration != 'Release') {
2592                             return
2593                         }
2594                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2595                             return
2596                         }
2597                     }
2598
2599                     // For RedHat and Debian, we only do Release builds.
2600                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2601                         if (configuration != 'Release') {
2602                             return
2603                         }
2604                         if (!isNormalOrInnerloop) {
2605                             return
2606                         }
2607                     }
2608
2609                     // Next, filter based on scenario.
2610
2611                     if (isJitStressScenario(scenario)) {
2612                         if (configuration != 'Checked') {
2613                             return
2614                         }
2615
2616                         // CoreFx JIT stress tests currently only implemented for ARM.
2617                         if (isCoreFxScenario(scenario) && (architecture != 'arm')) {
2618                             return
2619                         }
2620                     }
2621                     else if (isR2RBaselineScenario(scenario)) {
2622                         if (configuration != 'Checked' && configuration != 'Release') {
2623                             return
2624                         }
2625                     }
2626                     else if (isR2RStressScenario(scenario)) {
2627                         if (configuration != 'Checked') {
2628                             return
2629                         }
2630                     }
2631                     else {
2632                         // Skip scenarios
2633                         switch (scenario) {
2634                             case 'ilrt':
2635                             case 'longgc':
2636                             case 'gcsimulator':
2637                                 // Long GC tests take a long time on non-Release builds
2638                                 // ilrt is also Release only
2639                                 if (configuration != 'Release') {
2640                                     return
2641                                 }
2642                                 break
2643
2644                             case 'jitdiff':
2645                                 if (configuration != 'Checked') {
2646                                     return;
2647                                 }
2648                                 break
2649
2650                             case 'gc_reliability_framework':
2651                             case 'standalone_gc':
2652                                 if (configuration != 'Release' && configuration != 'Checked') {
2653                                     return
2654                                 }
2655                                 break
2656
2657                             case 'formatting':
2658                                 return
2659                             case 'illink':
2660                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2661                                     return
2662                                 }
2663                                 break
2664
2665                             case 'normal':
2666                                 // Nothing skipped
2667                                 break
2668
2669                             case 'innerloop':
2670                                 // Nothing skipped
2671                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2672                                     return
2673                                 }
2674                                 break
2675
2676                             default:
2677                                 println("Unknown scenario: ${scenario}")
2678                                 assert false
2679                                 break
2680                         }
2681                     }
2682
2683                     // Done filtering. Now, create the jobs.
2684
2685                     // =============================================================================================
2686                     // Create the test job
2687                     // =============================================================================================
2688
2689                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2690
2691                     def lowerConfiguration = configuration.toLowerCase()
2692                     def osGroup = getOSGroup(os)
2693                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2694
2695                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2696                     def inputCoreCLRBuildIsBuildOnly = false
2697                     if (isCoreFxScenario(scenario)) {
2698                         // Every CoreFx test depends on its own unique build.
2699                         inputCoreCLRBuildScenario = scenario
2700                         inputCoreCLRBuildIsBuildOnly = true
2701                     }
2702                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
2703                     def inputCoreCLRBuildName = projectFolder + '/' +
2704                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
2705
2706                     def inputWindowsTestsBuildName = ""
2707                     if (windowsArmJob != true) {
2708                         // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2709                         // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2710
2711                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2712
2713                         def inputWindowsTestBuildArch = architecture
2714                         if (architecture == "arm64" && os != "Windows_NT") {
2715                             // Use the x64 test build for arm64 unix
2716                             inputWindowsTestBuildArch = "x64"
2717                         }
2718
2719                         if (isJitStressScenario(scenario)) {
2720                             inputWindowsTestsBuildName = projectFolder + '/' +
2721                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2722                         } else {
2723                             inputWindowsTestsBuildName = projectFolder + '/' +
2724                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2725                         }
2726                     } // if (windowsArmJob != true)
2727
2728                     def serverGCString = ''
2729                     def testOpts = ''
2730
2731                     if (windowsArmJob != true) {
2732                         // Enable Server GC for Ubuntu PR builds
2733                         if (os == 'Ubuntu' && isPR) {
2734                             serverGCString = '--useServerGC'
2735                         }
2736
2737                         if (isR2RScenario(scenario)) {
2738
2739                             testOpts += ' --crossgen --runcrossgentests'
2740
2741                             if (scenario == 'r2r_jitstress1') {
2742                                 testOpts += ' --jitstress=1'
2743                             }
2744                             else if (scenario == 'r2r_jitstress2') {
2745                                 testOpts += ' --jitstress=2'
2746                             }
2747                             else if (scenario == 'r2r_jitstressregs1') {
2748                                 testOpts += ' --jitstressregs=1'
2749                             }
2750                             else if (scenario == 'r2r_jitstressregs2') {
2751                                 testOpts += ' --jitstressregs=2'
2752                             }
2753                             else if (scenario == 'r2r_jitstressregs3') {
2754                                 testOpts += ' --jitstressregs=3'
2755                             }
2756                             else if (scenario == 'r2r_jitstressregs4') {
2757                                 testOpts += ' --jitstressregs=4'
2758                             }
2759                             else if (scenario == 'r2r_jitstressregs8') {
2760                                 testOpts += ' --jitstressregs=8'
2761                             }
2762                             else if (scenario == 'r2r_jitstressregs0x10') {
2763                                 testOpts += ' --jitstressregs=0x10'
2764                             }
2765                             else if (scenario == 'r2r_jitstressregs0x80') {
2766                                 testOpts += ' --jitstressregs=0x80'
2767                             }
2768                             else if (scenario == 'r2r_jitstressregs0x1000') {
2769                                 testOpts += ' --jitstressregs=0x1000'
2770                             }
2771                             else if (scenario == 'r2r_jitminopts') {
2772                                 testOpts += ' --jitminopts'
2773                             }
2774                             else if (scenario == 'r2r_jitforcerelocs') {
2775                                 testOpts += ' --jitforcerelocs'
2776                             }
2777                             else if (scenario == 'r2r_gcstress15') {
2778                                 testOpts += ' --gcstresslevel=0xF'
2779                             }
2780                         }
2781                         else if (scenario == 'jitdiff') {
2782                             testOpts += ' --jitdisasm --crossgen'
2783                         }
2784                         else if (scenario == 'illink') {
2785                             testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2786                         }
2787                         else if (isLongGc(scenario)) {
2788                             // Long GC tests behave very poorly when they are not
2789                             // the only test running (many of them allocate until OOM).
2790                             testOpts += ' --sequential'
2791
2792                             // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2793                             // for running long GC and GCSimulator tests, respectively. We don't use them
2794                             // here because using a playlist file produces much more readable output on the CI machines
2795                             // and reduces running time.
2796                             //
2797                             // The Long GC playlist contains all of the tests that are
2798                             // going to be run. The GCSimulator playlist contains all of
2799                             // the GC simulator tests.
2800                             if (scenario == 'longgc') {
2801                                 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2802                             }
2803                             else if (scenario == 'gcsimulator') {
2804                                 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2805                             }
2806                         }
2807                         else if (isGcReliabilityFramework(scenario)) {
2808                             testOpts += ' --build-overlay-only'
2809                         }
2810                         else if (scenario == 'standalone_gc') {
2811                             if (osGroup == 'OSX') {
2812                                 testOpts += ' --gcname=libclrgc.dylib'
2813                             }
2814                             else if (osGroup == 'Linux') {
2815                                 testOpts += ' --gcname=libclrgc.so'
2816                             }
2817                             else {
2818                                 println("Unexpected OS group: ${osGroup} for os ${os}")
2819                                 assert false
2820                             }
2821                         }
2822                     } // if (windowsArmJob != true)
2823
2824                     def folder = getJobFolder(scenario)
2825                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2826                         // Add parameters for the inputs
2827
2828                         if (windowsArmJob == true) {
2829                             parameters {
2830                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2831                             }
2832                         }
2833                         else {
2834                             parameters {
2835                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
2836                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2837                             }
2838                         }
2839
2840                         steps {
2841                             // Set up the copies
2842
2843                             // Coreclr build containing the tests and mscorlib
2844                             // pri1 jobs still need to copy windows_nt built tests
2845                             if (windowsArmJob != true) {
2846                                 copyArtifacts(inputWindowsTestsBuildName) {
2847                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2848                                     buildSelector {
2849                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2850                                     }
2851                                 }
2852                             }
2853
2854                             // Coreclr build we are trying to test
2855                             //
2856                             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2857
2858                             copyArtifacts(inputCoreCLRBuildName) {
2859                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2860                                 buildSelector {
2861                                     buildNumber('${CORECLR_BUILD}')
2862                                 }
2863                             }
2864
2865                             // Windows CoreCLR Arm(64) will restore corefx
2866                             // packages correctly.
2867                             //
2868                             // In addition, test steps are entirely different
2869                             // because we do not have a unified runner
2870                             if (windowsArmJob != true) {
2871                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2872
2873                                 // HACK -- Arm64 does not have corefx jobs yet.
2874                                 // Clone corefx and build the native packages overwriting the x64 packages.
2875                                 if (architecture == 'arm64') {
2876                                     shell("mkdir -p ./bin/CoreFxBinDir")
2877                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2878                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2879                                 }
2880                                 else if (architecture == 'x86') {
2881                                     shell("mkdir ./bin/CoreFxNative")
2882
2883                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2884                                         includePatterns('bin/build.tar.gz')
2885                                         targetDirectory('bin/CoreFxNative')
2886                                         buildSelector {
2887                                             latestSuccessful(true)
2888                                         }
2889                                     }
2890
2891                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2892                                 }
2893
2894                                 // Unzip the tests first.  Exit with 0
2895                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2896                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2897
2898                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2899
2900                                 // Execute the tests
2901                                 def runDocker = isNeedDocker(architecture, os, false)
2902                                 def dockerPrefix = ""
2903                                 def dockerCmd = ""
2904                                 if (runDocker) {
2905                                     def dockerImage = getDockerImageName(architecture, os, false)
2906                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2907                                     dockerCmd = dockerPrefix + "${dockerImage} "
2908                                 }
2909
2910                                 // If we are running a stress mode, we'll set those variables first
2911                                 def testEnvOpt = ""
2912                                 if (isJitStressScenario(scenario)) {
2913                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2914                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2915                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2916                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2917                                     shell("${envScriptCmds}")
2918                                     testEnvOpt = "--test-env=" + scriptFileName
2919                                 }
2920
2921                                 if (isGCStressRelatedTesting(scenario)) {
2922                                     shell('./init-tools.sh')
2923                                 }
2924
2925                                 shell("""${dockerCmd}./tests/runtest.sh \\
2926                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2927                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2928                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2929                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2930
2931                                 if (isGcReliabilityFramework(scenario)) {
2932                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2933                                     if (serverGCString != '') {
2934                                         if (runDocker) {
2935                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2936                                         }
2937                                         else {
2938                                             shell("export COMPlus_gcServer=1")
2939                                         }
2940                                     }
2941
2942                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2943                                 }
2944                             }
2945                             else { // windowsArmJob == true
2946
2947                                 if (isCoreFxScenario(scenario)) {
2948
2949                                     // Only arm supported for corefx testing now.
2950                                     assert architecture == 'arm'
2951
2952                                     // Unzip CoreFx runtime
2953                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2954
2955                                     // Unzip CoreFx tests.
2956                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2957
2958                                     // Add the script to run the corefx tests
2959                                     def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2960                                     def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2961                                     def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2962                                     batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2963
2964                                 } else { // !isCoreFxScenario(scenario)
2965
2966                                     // Unzip tests.
2967                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2968
2969                                     def buildCommands = ""
2970
2971                                     def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2972                                     def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2973                                     def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2974
2975                                     // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2976                                     addCommand("SETLOCAL ENABLEEXTENSIONS")
2977
2978                                     // For all jobs
2979                                     addEnvVariable("CORE_ROOT", coreRootLocation)
2980
2981                                     addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2982                                     addEnvVariable("COMPlus_ContinueOnAssert", "0")
2983
2984                                     // ARM legacy backend; this is an altjit.
2985                                     if (architecture == "armlb") {
2986                                         addEnvVariable("COMPlus_AltJit", "*")
2987                                         addEnvVariable("COMPlus_AltJitNgen", "*")
2988                                         addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2989                                         addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2990                                     }
2991
2992                                     // If we are running a stress mode, we'll set those variables as well
2993                                     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2994                                         def stressValues = null
2995                                         if (isJitStressScenario(scenario)) {
2996                                             stressValues = Constants.jitStressModeScenarios[scenario]
2997                                         }
2998                                         else {
2999                                             stressValues = Constants.r2rStressScenarios[scenario]
3000                                         }
3001
3002                                         stressValues.each { key, value ->
3003                                             addEnvVariable(key, value)
3004                                         }
3005                                     }
3006
3007                                     if (isR2RScenario(scenario)) {
3008                                         // Crossgen the framework assemblies.
3009                                         buildCommands += """
3010 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3011 @goto skip_PrecompileAssembly
3012
3013 :PrecompileAssembly
3014 @REM Skip mscorlib since it is already precompiled.
3015 @if /I "%3" == "mscorlib.dll" exit /b 0
3016 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3017
3018 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3019 @if "%errorlevel%" == "-2146230517" (
3020     echo %2 is not a managed assembly.
3021 ) else if "%errorlevel%" == "-2146234344" (
3022     echo %2 is not a managed assembly.
3023 ) else if %errorlevel% neq 0 (
3024     echo Unable to precompile %2
3025 ) else (
3026     echo Precompiled %2
3027 )
3028 @exit /b 0
3029
3030 :skip_PrecompileAssembly
3031 """
3032
3033                                         // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3034                                         // crossgen on tests before running them.
3035                                         addEnvVariable("RunCrossGen", "true")
3036                                     } // isR2RScenario(scenario)
3037
3038                                     // Create the smarty command
3039                                     def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3040                                     def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3041                                     def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3042
3043                                     def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
3044
3045                                     if (architecture == "armlb") {
3046                                         addExclude("LEGACYJIT_FAIL")
3047                                     }
3048
3049                                     // Exclude tests based on scenario.
3050                                     Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3051                                         addArchSpecificExclude(architecture, excludeTag)
3052                                     }
3053
3054                                     // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
3055                                     if (scenario == 'innerloop') {
3056                                         addExclude("pri1")
3057                                     }
3058
3059                                     // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3060                                     // TODO: We should create a "long running" job that runs these with a longer timeout.
3061                                     addExclude("LONG_RUNNING")
3062
3063                                     smartyCommand += "/lstFile Tests.lst"
3064
3065                                     def testListArch = [
3066                                         'arm64': 'arm64',
3067                                         'arm': 'arm',
3068                                         'armlb': 'arm'
3069                                     ]
3070
3071                                     def archLocation = testListArch[architecture]
3072
3073                                     addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3074                                     addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3075                                     addCommand("${smartyCommand}")
3076
3077                                     // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3078                                     // However, we also need to remove all the variables that were set during this batch file, so we
3079                                     // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3080                                     // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3081                                     // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3082                                     // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3083                                     // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3084                                     addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3085
3086                                     // ZIP up the smarty output, no matter what the smarty result.
3087                                     addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3088
3089                                     addCommand("echo %errorlevel%")
3090                                     addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3091
3092                                     // Use the smarty errorlevel as the script errorlevel.
3093                                     addCommand("exit /b %__save_smarty_errorlevel%")
3094
3095                                     batchFile(buildCommands)
3096                                 } // non-corefx testing
3097                             } // windowsArmJob == true
3098                         } // steps
3099                     } // job
3100
3101                     addToViews(newJob, isPR, architecture, os)
3102
3103                     if (scenario == 'jitdiff') {
3104                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3105                     }
3106
3107                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3108                     if (os in ['Ubuntu']) {
3109                         SummaryBuilder summaries = new SummaryBuilder()
3110                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3111                         summaries.emit(newJob)
3112                     }
3113
3114                     def affinityOptions = null
3115
3116                     if (windowsArmJob == true) {
3117                         affinityOptions = [
3118                             "use_arm64_build_machine" : false
3119                         ]
3120                     }
3121
3122                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3123                         affinityOptions = [
3124                             "large_pages" : false
3125                         ]
3126                     }
3127
3128                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3129                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3130
3131                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3132
3133                     if (windowsArmJob != true) {
3134                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3135                     }
3136                     else {
3137                         if (!isCoreFxScenario(scenario)) {
3138                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3139
3140                             // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3141                             // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3142                             // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3143                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3144                         }
3145                     }
3146
3147                     // =============================================================================================
3148                     // Create a build flow to join together the build and tests required to run this test.
3149                     // =============================================================================================
3150
3151                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3152                     // Linux CoreCLR test
3153                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3154                     def fullTestJobName = projectFolder + '/' + newJob.name
3155                     // Add a reference to the input jobs for report purposes
3156                     JobReport.Report.addReference(inputCoreCLRBuildName)
3157                     if (windowsArmJob != true) {
3158                         JobReport.Report.addReference(inputWindowsTestsBuildName)
3159                     }
3160                     JobReport.Report.addReference(fullTestJobName)
3161                     def newFlowJob = null
3162
3163                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3164                         // Do not create the flow job for RHEL jobs.
3165                         return
3166                     }
3167
3168                     // For pri0 jobs we can build tests on unix
3169                     if (windowsArmJob) {
3170                         // For Windows arm jobs there is no reason to build a parallel test job.
3171                         // The product build supports building and archiving the tests.
3172
3173                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3174                         buildFlow("""\
3175 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3176
3177 // And then build the test build
3178 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3179 """)
3180                         }
3181                     }
3182                     else {
3183                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3184                         buildFlow("""\
3185 // Build the input jobs in parallel
3186 parallel (
3187 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3188 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3189 )
3190
3191 // And then build the test build
3192 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3193             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3194 """)
3195                         }
3196                     }
3197
3198                     addToViews(newFlowJob, isPR, architecture, os)
3199
3200                     // For the flow jobs set the machine affinity as x64 if an armarch.
3201                     def flowArch = architecture
3202
3203                     if (flowArch in validWindowsNTCrossArches) {
3204                         flowArch = 'x64'
3205                         affinityOptions = null
3206                     }
3207
3208                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3209                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3210                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3211                 } // configuration
3212             } // os
3213         } // architecture
3214     } // isPR
3215 } // scenario
3216
3217 JobReport.Report.generateJobReport(out)
3218
3219 // Make the call to generate the help job
3220 Utilities.createHelperJob(this, project, branch,
3221     "Welcome to the ${project} Repository",  // This is prepended to the help message
3222     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3223
3224 Utilities.addCROSSCheck(this, project, branch)