netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'jitincompletehwintrinsic'
 266                // 'jitx86hwintrinsicnoavx'
 267                // 'jitx86hwintrinsicnoavx2'
 268                // 'jitx86hwintrinsicnosimd'
 269                // 'jitnox86hwintrinsic'
 270                'corefx_baseline':                        [],    // corefx tests don't use smarty
 271                'corefx_minopts':                         [],    // corefx tests don't use smarty
 272                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 273                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 274                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 275                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 276                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 277                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 278                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 279                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 280                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 281                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 283                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 284                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 285                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 286                'heapverify1':                            [],
 287                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 288                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 290                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 293
 294                //
 295                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 296                //
 297
 298                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 299                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 317     ]
 318
 319     def static validLinuxArm64Scenarios = [
 320         'normal',
 321         'r2r',
 322         'innerloop',
 323         'gcstress0x3',
 324         'gcstress0xc'
 325     ]
 326
 327     def static configurationList = ['Debug', 'Checked', 'Release']
 328
 329     // This is the set of architectures
 330     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 331 }
 332
 333 // **************************************************************
 334 // Create some specific views
 335 //
 336 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 337 // views based on a single regular expression. These views will be generated by adding a
 338 // specific set of jobs to them.
 339 //
 340 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 341 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 342 // those are required, don't add them (which simplifies the view pages, as well).
 343 // **************************************************************
 344
 345 class Views {
 346     def static MergeJobView = null
 347     def static PeriodicJobView = null
 348     def static ArchitectureViews = [:]
 349     def static OSViews = [:]
 350 }
 351
 352 // MergeJobView: include all jobs that execute when a PR change is merged.
 353 Views.MergeJobView = listView('Merge') {
 354     recurse()
 355     columns {
 356         status()
 357         weather()
 358         name()
 359         lastSuccess()
 360         lastFailure()
 361         lastDuration()
 362         buildButton()
 363     }
 364 }
 365
 366 // PeriodicJobView: include all jobs that execute on a schedule
 367 Views.PeriodicJobView = listView('Periodic') {
 368     recurse()
 369     columns {
 370         status()
 371         weather()
 372         name()
 373         lastSuccess()
 374         lastFailure()
 375         lastDuration()
 376         buildButton()
 377     }
 378 }
 379
 380 // Create a view for non-PR jobs for each architecture.
 381 Constants.architectureList.each { architecture ->
 382     Views.ArchitectureViews[architecture] = listView(architecture) {
 383         recurse()
 384         columns {
 385             status()
 386             weather()
 387             name()
 388             lastSuccess()
 389             lastFailure()
 390             lastDuration()
 391             buildButton()
 392         }
 393     }
 394 }
 395
 396 // Create a view for non-PR jobs for each OS.
 397 Constants.osList.each { os ->
 398     // Don't create one for the special 'Windows_NT_BuildOnly'
 399     if (os == 'Windows_NT_BuildOnly') {
 400         return
 401     }
 402     Views.OSViews[os] = listView(os) {
 403         recurse()
 404         columns {
 405             status()
 406             weather()
 407             name()
 408             lastSuccess()
 409             lastFailure()
 410             lastDuration()
 411             buildButton()
 412         }
 413     }
 414 }
 415
 416 def static addToMergeView(def job) {
 417     Views.MergeJobView.with {
 418         jobs {
 419             name(job.name)
 420         }
 421     }
 422 }
 423
 424 def static addToPeriodicView(def job) {
 425     Views.PeriodicJobView.with {
 426         jobs {
 427             name(job.name)
 428         }
 429     }
 430 }
 431
 432 def static addToViews(def job, def isPR, def architecture, def os) {
 433     if (isPR) {
 434         // No views want PR jobs currently.
 435         return
 436     }
 437
 438     // Add to architecture view.
 439     Views.ArchitectureViews[architecture].with {
 440         jobs {
 441             name(job.name)
 442         }
 443     }
 444
 445     // Add to OS view.
 446     Views.OSViews[os].with {
 447         jobs {
 448             name(job.name)
 449         }
 450     }
 451 }
 452
 453 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 454     addToPeriodicView(job)
 455     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 456 }
 457
 458 def static addGithubPushTriggerHelper(def job) {
 459     addToMergeView(job)
 460     Utilities.addGithubPushTrigger(job)
 461 }
 462
 463
 464 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 465     assert os instanceof String
 466     assert architecture instanceof String
 467
 468     def armArches = ['arm', 'armlb', 'arm64']
 469     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 470
 471     if (!(architecture in armArches)) {
 472         assert options == null
 473         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 474
 475         return
 476     }
 477
 478     // This is an arm(64) job.
 479     //
 480     // There are several options.
 481     //
 482     // Windows_NT
 483     //
 484     // Arm32 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 486     // Arm32 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 488     //
 489     // Arm64 (Build) -> latest-arm64
 490     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 491     // Arm64 (Test)  -> arm64-windows_nt
 492     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 493     //
 494     // Ubuntu
 495     //
 496     // Arm32 (Build) -> arm-cross-latest
 497     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 498     // Arm32 (Test)  -> NYI Arch not supported
 499     //       |->
 500     //
 501     // Arm64 (Build) -> arm64-cross-latest
 502     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 503     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 504     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 505     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 506     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 507
 508     // This has to be a arm arch
 509     assert architecture in armArches
 510     if (os == "Windows_NT") {
 511         // Arm(64) Windows jobs share the same machines for now
 512         def isBuild = options['use_arm64_build_machine'] == true
 513
 514         if (isBuild == true) {
 515             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 516         } else {
 517             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 518         }
 519     } else {
 520         assert os != 'Windows_NT'
 521         assert os in supportedArmLinuxOs
 522
 523         if (architecture == 'arm' || architecture == 'armlb') {
 524             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 525         } else {
 526             // Arm64 Linux
 527             if (options['is_build_only'] == true) {
 528                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 529             } else {
 530                 // Arm64 Test Machines
 531                 if (options['large_pages'] == false) {
 532                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 533                 } else {
 534                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 535                 }
 536             }
 537         }
 538     }
 539 }
 540
 541 def static isGCStressRelatedTesting(def scenario) {
 542     // The 'r2r_gcstress15' scenario is a basic scenario.
 543     // Detect it and make it a GCStress related.
 544     if (scenario == 'r2r_gcstress15')
 545     {
 546         return true;
 547     }
 548
 549     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 550     def scenarioName = scenario.toLowerCase()
 551     def isGCStressTesting = false
 552     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 553         if (k in gcStressTestEnvVars) {
 554             isGCStressTesting = true;
 555         }
 556     }
 557     return isGCStressTesting
 558 }
 559
 560 def static isCoreFxScenario(def scenario) {
 561     def corefx_prefix = 'corefx_'
 562     if (scenario.length() < corefx_prefix.length()) {
 563         return false
 564     }
 565     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 566 }
 567
 568 def static isR2RBaselineScenario(def scenario) {
 569     return (scenario == 'r2r')
 570 }
 571
 572 def static isR2RStressScenario(def scenario) {
 573     return Constants.r2rStressScenarios.containsKey(scenario)
 574 }
 575
 576 def static isR2RScenario(def scenario) {
 577     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 578 }
 579
 580 def static isJitStressScenario(def scenario) {
 581     return Constants.jitStressModeScenarios.containsKey(scenario)
 582 }
 583
 584 def static isLongGc(def scenario) {
 585     return (scenario == 'longgc' || scenario == 'gcsimulator')
 586 }
 587
 588 def static isJitDiff(def scenario) {
 589     return (scenario == 'jitdiff')
 590 }
 591
 592 def static isGcReliabilityFramework(def scenario) {
 593     return (scenario == 'gc_reliability_framework')
 594 }
 595
 596 def static isArmWindowsScenario(def scenario) {
 597     return Constants.validArmWindowsScenarios.containsKey(scenario)
 598 }
 599
 600 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 601     if (isBuildOnly == true) {
 602         os = 'Windows_NT_buildOnly'
 603     }
 604
 605     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 606
 607     if (validOsPrTriggerArchConfigs == null) {
 608         return false
 609     }
 610
 611     if (validOsPrTriggerArchConfigs[architecture] != null) {
 612         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 613
 614         if (!(configuration in validOsPrTriggerConfigs)) {
 615             return false
 616         }
 617     } else {
 618         return false
 619     }
 620
 621     return true
 622 }
 623
 624 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 625     // 2 hours (120 minutes) is the default timeout
 626     def timeout = 120
 627     def innerLoop = (scenario == "innerloop")
 628
 629     if (!innerLoop) {
 630         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 631         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 632         timeout = 240
 633     }
 634
 635     if (!isBuildOnly) {
 636         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 637         if (isGCStressRelatedTesting(scenario)) {
 638             timeout = 4320
 639         }
 640         else if (isCoreFxScenario(scenario)) {
 641             timeout = 360
 642         }
 643         else if (isJitStressScenario(scenario)) {
 644             timeout = 300
 645         }
 646         else if (isR2RBaselineScenario(scenario)) {
 647             timeout = 240
 648         }
 649         else if (isLongGc(scenario)) {
 650             timeout = 1440
 651         }
 652         else if (isJitDiff(scenario)) {
 653             timeout = 240
 654         }
 655         else if (isGcReliabilityFramework(scenario)) {
 656             timeout = 1440
 657         }
 658         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 659             timeout = 240
 660         }
 661     }
 662
 663     if (configuration == 'Debug') {
 664         // Debug runs can be very slow. Add an hour.
 665         timeout += 60
 666     }
 667
 668         if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 669                 // AltJit runs compile all methods twice.
 670                 timeout *= 2
 671         }
 672
 673     // If we've changed the timeout from the default, set it in the job.
 674
 675     if (timeout != 120) {
 676         Utilities.setJobTimeout(newJob, timeout)
 677     }
 678 }
 679
 680 def static getJobFolder(def scenario) {
 681     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 682         return 'jitstress'
 683     }
 684     if (scenario == 'illink') {
 685         return 'illink'
 686     }
 687     return ''
 688 }
 689
 690 def static getStressModeDisplayName(def scenario) {
 691     def displayStr = ''
 692     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 693         def prefixLength = 'COMPlus_'.length()
 694         if (k.length() >= prefixLength) {
 695             def modeName = k.substring(prefixLength, k.length())
 696             displayStr += ' ' + modeName + '=' + v
 697         }
 698     }
 699
 700     if (isCoreFxScenario(scenario)) {
 701         displayStr = ('CoreFx ' + displayStr).trim()
 702     }
 703
 704     return displayStr
 705 }
 706
 707 def static getR2RDisplayName(def scenario) {
 708     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 709     def displayStr = scenario
 710     def prefixLength = 'r2r_'.length()
 711     if (displayStr.length() >= prefixLength) {
 712         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 713     } else if (scenario == 'r2r') {
 714         displayStr = "R2R"
 715     }
 716     return displayStr
 717 }
 718
 719 //
 720 // Functions to create an environment script.
 721 //      envScriptCreate -- initialize the script (call first)
 722 //      envScriptFinalize -- finalize the script (call last)
 723 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 724 //      envScriptAppendExistingScript -- append an existing script to the generated script
 725 //
 726 // Each script returns a string of commands. Concatenate all the strings together before
 727 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 728 //
 729
 730 // Initialize the environment setting script.
 731 def static envScriptCreate(def os, def stepScriptLocation) {
 732     def stepScript = ''
 733     if (os == 'Windows_NT') {
 734         stepScript += "echo Creating TestEnv script\r\n"
 735         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 736
 737         // Create at least an empty script.
 738         stepScript += "echo. > ${stepScriptLocation}\r\n"
 739     }
 740     else {
 741         stepScript += "echo Creating environment setting script\n"
 742         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 743     }
 744
 745     return stepScript
 746 }
 747
 748 // Generates the string for setting stress mode variables.
 749 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 750     def stepScript = ''
 751     if (os == 'Windows_NT') {
 752         stressModeVars.each{ k, v ->
 753             // Write out what we are writing to the script file
 754             stepScript += "echo Setting ${k}=${v}\r\n"
 755             // Write out the set itself to the script file`
 756             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 757         }
 758     }
 759     else {
 760         stressModeVars.each{ k, v ->
 761             // Write out what we are writing to the script file
 762             stepScript += "echo Setting ${k}=${v}\n"
 763             // Write out the set itself to the script file`
 764             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 765         }
 766     }
 767
 768     return stepScript
 769 }
 770
 771 // Append an existing script to an environment script.
 772 // Returns string of commands to do this.
 773 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 774     assert (os == 'Windows_NT')
 775     def stepScript = ''
 776
 777     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 778     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 779
 780     return stepScript
 781 }
 782
 783 // Finalize an environment setting script.
 784 // Returns string of commands to do this.
 785 def static envScriptFinalize(def os, def stepScriptLocation) {
 786     def stepScript = ''
 787
 788     if (os == 'Windows_NT') {
 789         // Display the resulting script. This is useful when looking at the output log file.
 790         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 791         stepScript += "type ${stepScriptLocation}\r\n"
 792     }
 793     else {
 794         stepScript += "chmod +x ${stepScriptLocation}\n"
 795     }
 796
 797     return stepScript
 798 }
 799
 800 def static isNeedDocker(def architecture, def os, def isBuild) {
 801     if (isBuild) {
 802         if (architecture == 'x86' && os == 'Ubuntu') {
 803             return true
 804         }
 805         else if (architecture == 'arm') {
 806             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 807                 return true
 808             }
 809         }
 810     }
 811     else {
 812         if (architecture == 'x86' && os == 'Ubuntu') {
 813             return true
 814         }
 815     }
 816     return false
 817 }
 818
 819 def static getDockerImageName(def architecture, def os, def isBuild) {
 820     // We must change some docker private images to official later
 821     if (isBuild) {
 822         if (architecture == 'x86' && os == 'Ubuntu') {
 823             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 824         }
 825         else if (architecture == 'arm') {
 826             if (os == 'Ubuntu') {
 827                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 828             }
 829             else if (os == 'Ubuntu16.04') {
 830                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 831             }
 832             else if (os == 'Tizen') {
 833                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 834             }
 835         }
 836     }
 837     else {
 838         if (architecture == 'x86' && os == 'Ubuntu') {
 839             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 840         }
 841     }
 842     println("Unknown architecture to use docker: ${architecture} ${os}");
 843     assert false
 844 }
 845
 846 // Calculates the name of the build job based on some typical parameters.
 847 //
 848 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 849     // If the architecture is x64, do not add that info into the build name.
 850     // Need to change around some systems and other builds to pick up the right builds
 851     // to do that.
 852
 853     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 854     if (isBuildOnly) {
 855         suffix += '_bld'
 856     }
 857     def baseName = ''
 858     switch (architecture) {
 859         case 'x64':
 860             if (scenario == 'normal') {
 861                 // For now we leave x64 off of the name for compatibility with other jobs
 862                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 863             }
 864             else if (scenario == 'formatting') {
 865                 // we don't care about the configuration for the formatting job. It runs all configs
 866                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 867             }
 868             else {
 869                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 870             }
 871             break
 872         case 'arm64':
 873             if (os.toLowerCase() == "windows_nt") {
 874                 // These are cross builds
 875                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 876             }
 877             else {
 878                 // Defaults to a small page size set of machines.
 879                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 880             }
 881             break
 882         case 'arm':
 883             // These are cross builds
 884             if (os == 'Tizen') {
 885                 // ABI: softfp
 886                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 887             }
 888             else {
 889                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 890             }
 891             break
 892         case 'armlb':
 893             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 894             break
 895         case 'x86':
 896         case 'x86_arm_altjit':
 897         case 'x64_arm64_altjit':
 898             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 899             break
 900         default:
 901             println("Unknown architecture: ${architecture}");
 902             assert false
 903             break
 904     }
 905
 906     return baseName + suffix
 907 }
 908
 909 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 910
 911     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 912     if (os == 'Windows_NT') {
 913         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 914             if (!(branch in Constants.WindowsArm64Branches)) {
 915                 return
 916             }
 917         }
 918     }
 919
 920     // Check scenario.
 921     switch (scenario) {
 922         case 'innerloop':
 923             break
 924         case 'normal':
 925             switch (architecture) {
 926                 case 'x64':
 927                 case 'x86':
 928                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 929                         addPeriodicTriggerHelper(job, '@daily')
 930                     }
 931                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 932                         addGithubPushTriggerHelper(job)
 933                     }
 934                     break
 935                 case 'arm':
 936                 case 'armlb':
 937                 case 'x86_arm_altjit':
 938                 case 'x64_arm64_altjit':
 939                     addGithubPushTriggerHelper(job)
 940                     break
 941                 case 'arm64':
 942                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 943                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 944                     break
 945                 default:
 946                     println("Unknown architecture: ${architecture}");
 947                     assert false
 948                     break
 949             }
 950             break
 951         case 'r2r':
 952             assert !(os in bidailyCrossList)
 953             // r2r gets a push trigger for checked/release
 954             if (configuration == 'Checked' || configuration == 'Release') {
 955                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 956                 if (architecture == 'x64' && os != 'OSX10.12') {
 957                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 958                     if (isFlowJob || os == 'Windows_NT') {
 959                         addGithubPushTriggerHelper(job)
 960                     }
 961                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 962                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 963                     if (isFlowJob) {
 964                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 965                     }
 966                 }
 967                 // For x86, only add per-commit jobs for Windows
 968                 else if (architecture == 'x86') {
 969                     if (os == 'Windows_NT') {
 970                         addGithubPushTriggerHelper(job)
 971                     }
 972                 }
 973                 // arm64 r2r jobs should only run daily.
 974                 else if (architecture == 'arm64') {
 975                     if (os == 'Windows_NT') {
 976                         addPeriodicTriggerHelper(job, '@daily')
 977                     }
 978                 }
 979             }
 980             break
 981         case 'r2r_jitstress1':
 982         case 'r2r_jitstress2':
 983         case 'r2r_jitstressregs1':
 984         case 'r2r_jitstressregs2':
 985         case 'r2r_jitstressregs3':
 986         case 'r2r_jitstressregs4':
 987         case 'r2r_jitstressregs8':
 988         case 'r2r_jitstressregs0x10':
 989         case 'r2r_jitstressregs0x80':
 990         case 'r2r_jitstressregs0x1000':
 991         case 'r2r_jitminopts':
 992         case 'r2r_jitforcerelocs':
 993         case 'r2r_gcstress15':
 994             assert !(os in bidailyCrossList)
 995
 996             // GCStress=C is currently not supported on OS X
 997             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 998                 break
 999             }
1000
1001             // GC Stress 15 r2r gets a push trigger for checked/release
1002             if (configuration == 'Checked' || configuration == 'Release') {
1003                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1004                 if (architecture == 'x64') {
1005                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1006                     if (isFlowJob || os == 'Windows_NT') {
1007                         // Add a weekly periodic trigger
1008                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1009                     }
1010                 }
1011                 // For x86, only add per-commit jobs for Windows
1012                 else if (architecture == 'x86') {
1013                     if (os == 'Windows_NT') {
1014                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1015                     }
1016                 }
1017             }
1018             break
1019         case 'longgc':
1020             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1021             assert configuration == 'Release'
1022             assert architecture == 'x64'
1023             addPeriodicTriggerHelper(job, '@daily')
1024             // TODO: Add once external email sending is available again
1025             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1026             break
1027         case 'gcsimulator':
1028             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1029             assert configuration == 'Release'
1030             assert architecture == 'x64'
1031             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1032             // TODO: Add once external email sending is available again
1033             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1034             break
1035         case 'standalone_gc':
1036             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1037             assert (configuration == 'Release' || configuration == 'Checked')
1038             // TODO: Add once external email sending is available again
1039             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1040             addPeriodicTriggerHelper(job, '@daily')
1041             break
1042         case 'gc_reliability_framework':
1043             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1044             assert (configuration == 'Release' || configuration == 'Checked')
1045             // Only triggered by phrase.
1046             break
1047         case 'ilrt':
1048             assert !(os in bidailyCrossList)
1049             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1050             if (architecture == 'x64' && configuration == 'Release') {
1051                 // We don't expect to see a job generated except in these scenarios
1052                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1053                 if (isFlowJob || os == 'Windows_NT') {
1054                     addPeriodicTriggerHelper(job, '@daily')
1055                 }
1056             }
1057             break
1058         case 'jitdiff':
1059             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1060             assert configuration == 'Checked'
1061             assert (architecture == 'x64' || architecture == 'x86')
1062             addGithubPushTriggerHelper(job)
1063             break
1064         case 'formatting':
1065             assert (os == 'Windows_NT' || os == "Ubuntu")
1066             assert architecture == 'x64'
1067             addGithubPushTriggerHelper(job)
1068             break
1069         case 'jitstressregs1':
1070         case 'jitstressregs2':
1071         case 'jitstressregs3':
1072         case 'jitstressregs4':
1073         case 'jitstressregs8':
1074         case 'jitstressregs0x10':
1075         case 'jitstressregs0x80':
1076         case 'jitstressregs0x1000':
1077         case 'minopts':
1078         case 'forcerelocs':
1079         case 'jitstress1':
1080         case 'jitstress2':
1081         case 'jitstress2_jitstressregs1':
1082         case 'jitstress2_jitstressregs2':
1083         case 'jitstress2_jitstressregs3':
1084         case 'jitstress2_jitstressregs4':
1085         case 'jitstress2_jitstressregs8':
1086         case 'jitstress2_jitstressregs0x10':
1087         case 'jitstress2_jitstressregs0x80':
1088         case 'jitstress2_jitstressregs0x1000':
1089         case 'tailcallstress':
1090         case 'jitsse2only':
1091         case 'jitnosimd':
1092         case 'jitnox86hwintrinsic':
1093         case 'jitincompletehwintrinsic':
1094         case 'jitx86hwintrinsicnoavx':
1095         case 'jitx86hwintrinsicnoavx2':
1096         case 'jitx86hwintrinsicnosimd':
1097         case 'corefx_baseline':
1098         case 'corefx_minopts':
1099         case 'corefx_jitstress1':
1100         case 'corefx_jitstress2':
1101         case 'corefx_jitstressregs1':
1102         case 'corefx_jitstressregs2':
1103         case 'corefx_jitstressregs3':
1104         case 'corefx_jitstressregs4':
1105         case 'corefx_jitstressregs8':
1106         case 'corefx_jitstressregs0x10':
1107         case 'corefx_jitstressregs0x80':
1108         case 'corefx_jitstressregs0x1000':
1109         case 'zapdisable':
1110             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1111                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1112                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1113                     if (os == 'Windows_NT') {
1114                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1115                         addPeriodicTriggerHelper(job, '@weekly')
1116                     }
1117                 }
1118                 else {
1119                     addPeriodicTriggerHelper(job, '@daily')
1120                 }
1121             }
1122             break
1123         case 'heapverify1':
1124         case 'gcstress0x3':
1125             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1126                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1127                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1128                     if (os == 'Windows_NT') {
1129                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1130                         addPeriodicTriggerHelper(job, '@weekly')
1131                     }
1132                     // TODO: Add once external email sending is available again
1133                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1134                 }
1135                 else {
1136                     addPeriodicTriggerHelper(job, '@weekly')
1137                 }
1138             }
1139             break
1140         case 'gcstress0xc':
1141         case 'gcstress0xc_zapdisable':
1142         case 'gcstress0xc_zapdisable_jitstress2':
1143         case 'gcstress0xc_zapdisable_heapverify1':
1144         case 'gcstress0xc_jitstress1':
1145         case 'gcstress0xc_jitstress2':
1146         case 'gcstress0xc_minopts_heapverify1':
1147             // GCStress=C is currently not supported on OS X
1148             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1149                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1150                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1151                     if (os == 'Windows_NT') {
1152                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1153                         addPeriodicTriggerHelper(job, '@weekly')
1154                     }
1155                     // TODO: Add once external email sending is available again
1156                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1157                 }
1158                 else {
1159                     addPeriodicTriggerHelper(job, '@weekly')
1160                 }
1161             }
1162             break
1163
1164         case 'illink':
1165             // Testing on other operating systems TBD
1166             assert (os == 'Windows_NT' || os == 'Ubuntu')
1167             if (architecture == 'x64' || architecture == 'x86') {
1168                 if (configuration == 'Checked') {
1169                     addPeriodicTriggerHelper(job, '@daily')
1170                 }
1171             }
1172             break
1173
1174         case 'tieredcompilation':
1175         case 'corefx_tieredcompilation':
1176             // No periodic jobs just yet, still testing
1177             break
1178
1179         default:
1180             println("Unknown scenario: ${scenario}");
1181             assert false
1182             break
1183     }
1184     return
1185 }
1186
1187 // **************************
1188 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1189 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1190 // builds will do a couple extra steps.
1191 // **************************
1192
1193 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1194 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1195 // it's just used for internal builds
1196 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1197 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1198     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1199
1200     if (isWindowsBuildOnlyJob) {
1201         return
1202     }
1203
1204     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1205     // Non pull request builds.
1206     if (!isPR) {
1207         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1208         return
1209     }
1210
1211      def arm64Users = [
1212         'AndyAyersMS',
1213         'briansull',
1214         'BruceForstall',
1215         'CarolEidt',
1216         'cmckinsey',
1217         'erozenfeld',
1218         'janvorli',
1219         'jashook',
1220         'JosephTremoulet',
1221         'pgodeq',
1222         'pgavlin',
1223         'rartemev',
1224         'russellhadley',
1225         'RussKeldorph',
1226         'sandreenko',
1227         'sdmaclea',
1228         'swaroop-sridhar',
1229         'jkotas',
1230         'markwilkie',
1231         'weshaggard'
1232     ]
1233
1234     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1235     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1236     def osGroup = getOSGroup(os)
1237     switch (architecture) {
1238         case 'x64': // editor brace matching: {
1239             if (scenario == 'formatting') {
1240                 assert configuration == 'Checked'
1241                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1242                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1243                 }
1244
1245                 break
1246             }
1247
1248             switch (os) {
1249                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1250                 case 'Debian8.4':
1251                 case 'RHEL7.2':
1252                     if (scenario == 'innerloop') {
1253                         assert !isFlowJob
1254                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1255                     }
1256                     else if (scenario == 'normal') {
1257                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1258                     }
1259                     break
1260
1261                 case 'Ubuntu16.04':
1262                     assert !isFlowJob
1263                     assert scenario != 'innerloop'
1264                     // Distinguish with the other architectures (arm and x86)
1265                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1266                     break
1267
1268                 case 'Fedora24':
1269                 case 'Ubuntu16.10':
1270                     assert !isFlowJob
1271                     assert scenario != 'innerloop'
1272                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1273                     break
1274
1275                 case 'Ubuntu':
1276                     if (scenario == 'illink') {
1277                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                         break
1279                     }
1280                     // fall through
1281
1282                 case 'OSX10.12':
1283                     // Triggers on the non-flow jobs aren't necessary here
1284                     // Corefx testing uses non-flow jobs.
1285                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1286                         break
1287                     }
1288                     switch (scenario) {
1289                         case 'innerloop':
1290                             // PR Triggered jobs. These jobs will run pri0 tests.
1291                             if (configuration == 'Checked') {
1292                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1293                             }
1294                             break
1295
1296                         case 'normal':
1297                             // OSX uses checked for default PR tests
1298                             if (configuration == 'Checked') {
1299                                 // Default trigger
1300                                 assert !job.name.contains("centos")
1301                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1302                             }
1303                             break
1304
1305                         case 'jitdiff':
1306                             if (configuration == 'Checked') {
1307                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1308                             }
1309                             break
1310
1311                         case 'ilrt':
1312                             if (configuration == 'Release') {
1313                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1314                             }
1315                             break
1316
1317                         case 'longgc':
1318                             if (configuration == 'Release') {
1319                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1320                             }
1321                             break
1322
1323                         case 'gcsimulator':
1324                             if (configuration == 'Release') {
1325                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1326                             }
1327                             break
1328
1329                         case 'standalone_gc':
1330                             if (configuration == 'Release' || configuration == 'Checked') {
1331                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1332                             }
1333                             break
1334
1335                         case 'gc_reliability_framework':
1336                             if (configuration == 'Release' || configuration == 'Checked') {
1337                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1338                             }
1339                             break
1340
1341                         default:
1342                             if (isJitStressScenario(scenario)) {
1343                                 def displayStr = getStressModeDisplayName(scenario)
1344                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1345                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1346                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1347                             }
1348                             else if (isR2RScenario(scenario)) {
1349                                 if (configuration == 'Release' || configuration == 'Checked') {
1350                                     def displayStr = getR2RDisplayName(scenario)
1351                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1352                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1353                                 }
1354                             }
1355                             else {
1356                                 println("Unknown scenario: ${scenario}");
1357                                 assert false
1358                             }
1359                             break
1360
1361                     }
1362                     break
1363
1364                 case 'CentOS7.1':
1365                     switch (scenario) {
1366                         case 'innerloop':
1367                             // CentOS uses checked for default PR tests while debug is build only
1368                             if (configuration == 'Debug') {
1369                                 // Default trigger
1370                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1371                             }
1372
1373                             // Make sure this is a flow job to get build and test.
1374                             if (configuration == 'Checked' && isFlowJob) {
1375                                 assert job.name.contains("flow")
1376                                 // Default trigger
1377                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1378                             }
1379                             break
1380
1381                         case 'normal':
1382                             // Make sure this is a flow job to get build and test.
1383                             if (configuration == 'Checked' && isFlowJob) {
1384                                 assert job.name.contains("flow")
1385                                 // Default trigger
1386                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1387                             }
1388                             break
1389
1390                         default:
1391                             if (isR2RScenario(scenario)) {
1392                                 if (configuration == 'Release' || configuration == 'Checked') {
1393                                     def displayStr = getR2RDisplayName(scenario)
1394                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1395                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1396                                 }
1397                             }
1398                             break
1399
1400                     }
1401                     break
1402
1403                 case 'Windows_NT':
1404                     switch (scenario) {
1405                         case 'innerloop':
1406                             // Default trigger
1407                             if (configuration == 'Checked' || configuration == 'Release') {
1408                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1409                             }
1410                             break
1411
1412                         case 'normal':
1413                             if (configuration == 'Checked') {
1414                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1415                             }
1416                             break
1417
1418                         case 'jitdiff':
1419                             if (configuration == 'Checked') {
1420                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1421                             }
1422                             break
1423
1424                         case 'ilrt':
1425                             if (configuration == 'Release') {
1426                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1427                             }
1428                             break
1429
1430                         case 'longgc':
1431                             if (configuration == 'Release') {
1432                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1433                             }
1434                             break
1435
1436                         case 'gcsimulator':
1437                             if (configuration == 'Release') {
1438                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1439                             }
1440                             break
1441
1442                         case 'standalone_gc':
1443                             if (configuration == 'Release' || configuration == 'Checked') {
1444                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1445                             }
1446                             break
1447
1448                         case 'gc_reliability_framework':
1449                             if (configuration == 'Release' || configuration == 'Checked') {
1450                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1451                             }
1452                             break
1453
1454                         case 'illink':
1455                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1456                             break
1457
1458                         default:
1459                             if (isJitStressScenario(scenario)) {
1460                                 def displayStr = getStressModeDisplayName(scenario)
1461                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1462                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1463                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1464                             }
1465                             else if (isR2RScenario(scenario)) {
1466                                 if (configuration == 'Release' || configuration == 'Checked') {
1467                                     def displayStr = getR2RDisplayName(scenario)
1468                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1469                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1470                                 }
1471                             }
1472                             else {
1473                                 println("Unknown scenario: ${scenario}");
1474                                 assert false
1475                             }
1476                             break
1477
1478                     }
1479                     break
1480
1481                 default:
1482                     println("Unknown os: ${os}");
1483                     assert false
1484                     break
1485
1486             }
1487
1488             break
1489
1490         // editor brace matching: }
1491         case 'armlb':
1492         case 'arm': // editor brace matching: {
1493             switch (os) {
1494                 case 'Ubuntu':
1495                 case 'Ubuntu16.04':
1496                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1497                         break
1498                     }
1499
1500                     assert scenario != 'innerloop'
1501                     job.with {
1502                         publishers {
1503                             azureVMAgentPostBuildAction {
1504                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1505                             }
1506                         }
1507                     }
1508                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1509                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1510                     break
1511
1512                 case 'Tizen':
1513                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1514                         break
1515                     }
1516
1517                     architecture = 'armel'
1518                     job.with {
1519                         publishers {
1520                             azureVMAgentPostBuildAction {
1521                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1522                             }
1523                         }
1524                     }
1525
1526                     if (scenario == 'innerloop') {
1527                         if (configuration == 'Checked') {
1528                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1529                         }
1530                     }
1531                     else {
1532                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1533                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1534                     }
1535                     break
1536
1537                 case 'Windows_NT':
1538                     // Triggers on the non-flow jobs aren't necessary here
1539                     if (!isFlowJob) {
1540                         break
1541                     }
1542
1543                     // Set up a private trigger
1544                     def contextString = "${os} ${architecture} Cross ${configuration}"
1545                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1546                     if (scenario == 'innerloop') {
1547                         contextString += " Innerloop"
1548                         triggerString += "\\W+Innerloop"
1549                     }
1550                     else {
1551                         contextString += " ${scenario}"
1552                         triggerString += "\\W+${scenario}"
1553                     }
1554
1555                     if (configuration == 'Debug') {
1556                         contextString += " Build"
1557                         triggerString += "\\W+Build"
1558                     } else {
1559                         contextString += " Build and Test"
1560                         triggerString += "\\W+Build and Test"
1561                     }
1562
1563                     triggerString += ".*"
1564
1565                     switch (scenario) {
1566                         case 'innerloop':
1567                             // Only Checked is an innerloop trigger.
1568                             if (configuration == 'Checked')
1569                             {
1570                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1571                             }
1572                             break
1573                         case 'normal':
1574                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1575                             break
1576                         default:
1577                             // Stress jobs will use this code path.
1578                             if (isArmWindowsScenario(scenario)) {
1579                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1580                             }
1581                             break
1582                     }
1583                     break
1584                 default:
1585                     println("NYI os: ${os}");
1586                     assert false
1587                     break
1588             }
1589             break
1590         // editor brace matching: }
1591         case 'arm64': // editor brace matching: {
1592             // Set up a private trigger
1593             def contextString = "${os} ${architecture} Cross ${configuration}"
1594             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1595
1596             if (scenario == 'innerloop') {
1597                 contextString += " Innerloop"
1598                 triggerString += "\\W+Innerloop"
1599             }
1600             else {
1601                 contextString += " ${scenario}"
1602                 triggerString += "\\W+${scenario}"
1603             }
1604
1605             if (configuration == 'Debug') {
1606                 contextString += " Build"
1607                 triggerString += "\\W+Build"
1608             } else {
1609                 contextString += " Build and Test"
1610                 triggerString += "\\W+Build and Test"
1611             }
1612
1613             triggerString += ".*"
1614
1615             switch (os) {
1616                 case 'Ubuntu':
1617                 case 'Ubuntu16.04':
1618                     switch (scenario) {
1619                         case 'innerloop':
1620                             if (configuration == 'Debug' && !isFlowJob) {
1621                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1622                             }
1623
1624                             break
1625                         case 'normal':
1626                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1627                             break
1628                         default:
1629                             if (isR2RScenario(scenario)) {
1630                                 if (configuration == 'Checked' || configuration == 'Release') {
1631                                     def displayStr = getR2RDisplayName(scenario)
1632                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1633                                 }
1634                             }
1635                             break
1636                     }
1637                     break
1638
1639                 case 'Windows_NT':
1640                     // Triggers on the non-flow jobs aren't necessary here
1641                     if (!isFlowJob) {
1642                         break
1643                     }
1644
1645                     assert isArmWindowsScenario(scenario)
1646                     switch (scenario) {
1647                         case 'innerloop':
1648                             if (configuration == 'Checked') {
1649                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1650                             }
1651
1652                             break
1653                         case 'normal':
1654                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1655                             break
1656                         default:
1657                             // Stress jobs will use this code path.
1658                             if (isArmWindowsScenario(scenario)) {
1659                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1660                             }
1661                             break
1662                     }
1663                     break
1664                 default:
1665                     println("NYI os: ${os}");
1666                     assert false
1667                     break
1668             }
1669             break
1670
1671         // editor brace matching: }
1672         case 'x86': // editor brace matching: {
1673             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1674             if (os == 'Ubuntu') {
1675                 // Triggers on the non-flow jobs aren't necessary here
1676                 if (!isFlowJob) {
1677                     break
1678                 }
1679
1680                 // on-demand only for ubuntu x86
1681                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1682                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1683                 break
1684
1685             }
1686             switch (scenario) {
1687                 case 'innerloop':
1688                     if (configuration == 'Checked' || configuration == 'Release') {
1689                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1690                     }
1691                     break
1692
1693                 case 'normal':
1694                     if (configuration == 'Checked') {
1695                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1696                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1697                     }
1698                     break
1699
1700                 case 'ilrt':
1701                     if (configuration == 'Release') {
1702                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1703                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1704                     }
1705                     break
1706
1707                 case 'longgc':
1708                     if (configuration == 'Release') {
1709                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1710                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1711                     }
1712                     break
1713
1714                 case 'gcsimulator':
1715                     if (configuration == 'Release') {
1716                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1717                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1718                     }
1719                     break
1720
1721                 case 'standalone_gc':
1722                     if (configuration == 'Release' || configuration == 'Checked') {
1723                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1724                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1725                     }
1726                     break
1727
1728                 case 'illink':
1729                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1730                     break
1731
1732                 default:
1733                     if (isJitStressScenario(scenario)) {
1734                         def displayStr = getStressModeDisplayName(scenario)
1735                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1736                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1737                     }
1738                     else if (isR2RScenario(scenario)) {
1739                         if (configuration == 'Release' || configuration == 'Checked') {
1740                             def displayStr = getR2RDisplayName(scenario)
1741                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1742                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1743                         }
1744                     }
1745                     else {
1746                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1747                         assert false
1748                     }
1749                     break
1750
1751             }
1752             break
1753
1754          // editor brace matching: }
1755         case 'x64_arm64_altjit':
1756         case 'x86_arm_altjit': // editor brace matching: {
1757             assert (os == 'Windows_NT')
1758             switch (scenario) {
1759                 case 'normal':
1760                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1761                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1762                     break
1763                 default:
1764                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1765                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1766                     break
1767             }
1768             break
1769
1770         // editor brace matching: }
1771         default:
1772             println("Unknown architecture: ${architecture}");
1773             assert false
1774             break
1775     }
1776 }
1777
1778 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1779     def buildCommands = [];
1780     def osGroup = getOSGroup(os)
1781     def lowerConfiguration = configuration.toLowerCase()
1782
1783     def priority = '1'
1784     if (scenario == 'innerloop') {
1785         priority = '0'
1786     }
1787
1788     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1789
1790     def enableCorefxTesting = isCoreFxScenario(scenario)
1791
1792     // Calculate the build steps, archival, and xunit results
1793     switch (os) {
1794         case 'Windows_NT': // editor brace matching: {
1795             switch (architecture) {
1796                 case 'x64':
1797                 case 'x86':
1798                 case 'x86_arm_altjit':
1799                 case 'x64_arm64_altjit':
1800                     def arch = architecture
1801                     def buildOpts = ''
1802                     if (architecture == 'x86_arm_altjit') {
1803                         arch = 'x86'
1804                     }
1805                     else if (architecture == 'x64_arm64_altjit') {
1806                         arch = 'x64'
1807                     }
1808
1809                     if (scenario == 'formatting') {
1810                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1811                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1812                         break
1813                     }
1814
1815                     if (scenario == 'illink') {
1816                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1817                     }
1818
1819                     // If it is a release build for windows, ensure PGO is used, else fail the build
1820                     if ((lowerConfiguration == 'release') &&
1821                         (scenario in Constants.basicScenarios) &&
1822                         (architecture != 'x86_arm_altjit') &&
1823                         (architecture != 'x64_arm64_altjit')) {
1824
1825                         buildOpts += ' -enforcepgo'
1826                     }
1827
1828                     if (enableCorefxTesting) {
1829                         buildOpts += ' skiptests';
1830                     } else {
1831                         buildOpts += " -priority=${priority}"
1832                     }
1833
1834                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1835                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1836                     // 35 characters long.
1837
1838                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1839
1840                     if (!isBuildOnly) {
1841                         def runtestArguments = ''
1842                         def testOpts = 'collectdumps'
1843
1844                         if (isR2RScenario(scenario)) {
1845
1846                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1847                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1848                             // to cause the tests to be crossgen'ed.
1849
1850                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1851                                 testOpts += ' crossgenaltjit protononjit.dll'
1852                             } else {
1853                                 testOpts += ' crossgen'
1854                             }
1855
1856                             testOpts += ' runcrossgentests'
1857                         }
1858                         else if (scenario == 'jitdiff') {
1859                             testOpts += ' jitdisasm crossgen'
1860                         }
1861                         else if (scenario == 'ilrt') {
1862                             testOpts += ' ilasmroundtrip'
1863                         }
1864                         else if (isLongGc(scenario)) {
1865                             testOpts += " ${scenario} sequential"
1866                         }
1867                         else if (scenario == 'standalone_gc') {
1868                             testOpts += ' gcname clrgc.dll'
1869                         }
1870                         else if (scenario == 'illink') {
1871                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1872                         }
1873
1874                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1875                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1876                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1877                         // which is read by the xunit harness.)
1878                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1879                         {
1880                             def timeout = 1800000
1881                             testOpts += " timeout ${timeout}"
1882                         }
1883
1884                         // If we are running a stress mode, we should write out the set of key
1885                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1886
1887                         def envScriptPath = ''
1888                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1889                             def buildCommandsStr = ''
1890                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1891                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1892
1893                             if (isJitStressScenario(scenario)) {
1894                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1895                             }
1896                             else if (isR2RStressScenario(scenario)) {
1897                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1898                             }
1899
1900                             if (architecture == 'x86_arm_altjit') {
1901                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1902                             }
1903                             else if (architecture == 'x64_arm64_altjit') {
1904                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1905                             }
1906
1907                             envScriptFinalize(os, envScriptPath)
1908
1909                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1910                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1911                             // into a single script.
1912                             buildCommands += buildCommandsStr
1913                         }
1914                         else if (architecture == 'x86_arm_altjit') {
1915                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1916                         }
1917                         else if (architecture == 'x64_arm64_altjit') {
1918                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1919                         }
1920                         if (envScriptPath != '') {
1921                             testOpts += " TestEnv ${envScriptPath}"
1922                         }
1923
1924                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1925
1926                         if (enableCorefxTesting) {
1927                             def workspaceRelativeFxRoot = "_/fx"
1928                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1929
1930                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1931
1932                             // Archive and process (only) the test results
1933                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1934                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1935
1936                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1937                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1938                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1939                         }
1940                         else if (isGcReliabilityFramework(scenario)) {
1941                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1942                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1943                         }
1944                         else {
1945                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1946                         }
1947                     }
1948
1949                     if (!enableCorefxTesting) {
1950                         // Run the rest of the build
1951                         // Build the mscorlib for the other OS's
1952                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1953                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1954
1955                         if (arch == "x64") {
1956                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1957                         }
1958
1959                         // Zip up the tests directory so that we don't use so much space/time copying
1960                         // 10s of thousands of files around.
1961                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1962
1963                         if (!isJitStressScenario(scenario)) {
1964                             // For Windows, pull full test results and test drops for x86/x64.
1965                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1966                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1967                         }
1968
1969                         if (scenario == 'jitdiff') {
1970                             // retrieve jit-dasm output for base commit, and run jit-diff
1971                             if (!isBuildOnly) {
1972                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1973                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1974                             }
1975                         }
1976
1977                         if (!isBuildOnly) {
1978                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1979                         }
1980                     }
1981                     break
1982                 case 'armlb':
1983                 case 'arm':
1984                     assert isArmWindowsScenario(scenario)
1985
1986                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1987                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1988
1989                     def buildArchitecture = 'arm'
1990
1991                     def buildOpts = ''
1992
1993                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1994                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1995                     if (architecture == 'armlb') {
1996                         buildOpts += ' -crossgenaltjit legacyjit.dll'
1997                     }
1998
1999                     if (enableCorefxTesting) {
2000                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2001                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2002                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2003                         //
2004                         // buildOpts += ' skiptests';
2005                         buildOpts += " -priority=0"
2006                     } else {
2007                         buildOpts += " -priority=${priority}"
2008                     }
2009
2010                     // This is now a build only job. Do not run tests. Use the flow job.
2011                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2012
2013                     if (enableCorefxTesting) {
2014                         assert isBuildOnly
2015                         assert architecture == 'arm'
2016
2017                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2018                         // to determine the correct matching corefx version git commit hash.
2019                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2020
2021                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2022                         def envScriptPath = ''
2023                         def buildCommandsStr = ''
2024                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2025                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2026                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2027                         envScriptFinalize(os, envScriptPath)
2028                         buildCommands += buildCommandsStr
2029
2030                         def workspaceRelativeFxRootLinux = "_/fx"
2031                         def workspaceRelativeFxRootWin = "_\\fx"
2032                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2033
2034                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2035
2036                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2037                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2038                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2039
2040                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2041                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2042                     } else {
2043                         // Zip up the tests directory so that we don't use so much space/time copying
2044                         // 10s of thousands of files around.
2045                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2046
2047                         // Add archival.
2048                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2049                     }
2050                     break
2051                 case 'arm64':
2052                     assert isArmWindowsScenario(scenario)
2053
2054                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2055                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2056
2057                     // This is now a build only job. Do not run tests. Use the flow job.
2058                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2059
2060                     // Zip up the tests directory so that we don't use so much space/time copying
2061                     // 10s of thousands of files around.
2062                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2063
2064                     // Add archival.
2065                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2066                     break
2067                 default:
2068                     println("Unknown architecture: ${architecture}");
2069                     assert false
2070                     break
2071             }
2072             break
2073         // editor brace matching: }
2074         case 'Ubuntu':
2075         case 'Ubuntu16.04':
2076         case 'Ubuntu16.10':
2077         case 'Debian8.4':
2078         case 'OSX10.12':
2079         case 'CentOS7.1':
2080         case 'RHEL7.2':
2081         case 'Tizen':
2082         case 'Fedora24': // editor brace matching: {
2083             switch (architecture) {
2084                 case 'x64':
2085                 case 'x86':
2086                     if (architecture == 'x86' && os == 'Ubuntu') {
2087                         // build and PAL test
2088                         def dockerImage = getDockerImageName(architecture, os, true)
2089                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2090                         dockerImage = getDockerImageName(architecture, os, false)
2091                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2092                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2093                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2094                         break
2095                     }
2096
2097                     if (scenario == 'formatting') {
2098                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2099                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2100                         break
2101                     }
2102
2103                     if (scenario == 'illink') {
2104                         assert(os == 'Ubuntu')
2105                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2106                     }
2107
2108                     if (!enableCorefxTesting) {
2109                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2110                         // only on supported OS platforms.
2111                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2112                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2113
2114                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2115                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2116
2117                         // Basic archiving of the build
2118                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2119                         // And pal tests
2120                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2121                     }
2122                     else {
2123                         // Corefx stress testing
2124                         assert os == 'Ubuntu'
2125                         assert architecture == 'x64'
2126                         assert lowerConfiguration == 'checked'
2127                         assert isJitStressScenario(scenario)
2128
2129                         // Build coreclr
2130                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2131
2132                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2133
2134                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2135                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2136                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2137                         buildCommands += envScriptCmds
2138
2139                         // Build and text corefx
2140                         def workspaceRelativeFxRoot = "_/fx"
2141                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2142
2143                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2144
2145                         // Archive and process (only) the test results
2146                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2147                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2148                     }
2149                     break
2150                 case 'arm64':
2151                     if (!enableCorefxTesting) {
2152                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2153
2154                         // HACK -- Arm64 does not have corefx jobs yet.
2155                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2156                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2157                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2158                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2159
2160                         // Basic archiving of the build
2161                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2162                     }
2163                     break
2164                 case 'arm':
2165                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2166                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2167
2168                     // default values for Ubuntu
2169                     def arm_abi="arm"
2170                     def linuxCodeName="trusty"
2171                     if (os == 'Ubuntu16.04') {
2172                         linuxCodeName="xenial"
2173                     }
2174                     else if (os == 'Tizen') {
2175                         arm_abi="armel"
2176                         linuxCodeName="tizen"
2177                     }
2178
2179                     // Unzip the Windows test binaries first. Exit with 0
2180                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2181
2182                     // Unpack the corefx binaries
2183                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2184                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2185                     if (os != 'Tizen') {
2186                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2187                     }
2188                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2189                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2190                     if (os == 'Ubuntu') {
2191                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2192                     }
2193
2194                     // Call the ARM CI script to cross build and test using docker
2195                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2196                     --mode=docker \\
2197                     --${arm_abi} \\
2198                     --linuxCodeName=${linuxCodeName} \\
2199                     --buildConfig=${lowerConfiguration} \\
2200                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2201                     --coreFxBinDir=./bin/CoreFxBinDir \\
2202                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2203
2204                     // Basic archiving of the build, no pal tests
2205                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2206                     break
2207                 default:
2208                     println("Unknown architecture: ${architecture}");
2209                     assert false
2210                     break
2211             }
2212             break
2213         // editor brace matching: }
2214         default:
2215             println("Unknown os: ${os}");
2216             assert false
2217             break
2218     } // os
2219
2220     return buildCommands
2221 }
2222
2223 Constants.allScenarios.each { scenario ->
2224     [true, false].each { isPR ->
2225         Constants.architectureList.each { architecture ->
2226             Constants.configurationList.each { configuration ->
2227                 Constants.osList.each { os ->
2228                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2229                     // and reset the os to Windows_NT
2230                     def isBuildOnly = false
2231                     if (os == 'Windows_NT_BuildOnly') {
2232                         isBuildOnly = true
2233                         os = 'Windows_NT'
2234                     }
2235
2236                     // Tizen is only supported for arm architecture
2237                     if (os == 'Tizen' && architecture != 'arm') {
2238                         return
2239                     }
2240
2241                     // Skip totally unimplemented (in CI) configurations.
2242                     switch (architecture) {
2243                         case 'arm64':
2244                             if (os == 'Ubuntu16.04') {
2245                                 os = 'Ubuntu'
2246                             }
2247
2248                             // Windows and Ubuntu only
2249                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2250                                 return
2251                             }
2252                             break
2253                         case 'arm':
2254                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2255                                 return
2256                             }
2257                             break
2258                         case 'armlb':
2259                             if (os != 'Windows_NT') {
2260                                 return
2261                             }
2262                             break
2263                         case 'x86':
2264                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2265                                 return
2266                             }
2267                             break
2268                         case 'x86_arm_altjit':
2269                         case 'x64_arm64_altjit':
2270                             if (os != 'Windows_NT') {
2271                                 return
2272                             }
2273                             break
2274                         case 'x64':
2275                             // Everything implemented
2276                             break
2277                         default:
2278                             println("Unknown architecture: ${architecture}")
2279                             assert false
2280                             break
2281                     }
2282
2283                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2284                     // with checked builds)
2285                     if (isJitStressScenario(scenario)) {
2286                         if (configuration != 'Checked') {
2287                             return
2288                         }
2289
2290                         // Since these are just execution time differences,
2291                         // skip platforms that don't execute the tests here (Windows_NT only)
2292                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2293                         if (!isEnabledOS) {
2294                             return
2295                         }
2296
2297                         switch (architecture) {
2298                             case 'x64':
2299                             case 'x86':
2300                             case 'x86_arm_altjit':
2301                             case 'x64_arm64_altjit':
2302                                 // x86 ubuntu: default only
2303                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2304                                     return
2305                                 }
2306                                 if (isBuildOnly) {
2307                                     return
2308                                 }
2309                                 break
2310
2311                             case 'arm':
2312                                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2313                                 if (!isBuildOnly || !isCoreFxScenario(scenario)) {
2314                                     return
2315                                 }
2316                                 break
2317
2318                             default:
2319                                 // arm64, armlb: stress is handled through flow jobs.
2320                                 return
2321                         }
2322                     }
2323                     else if (isR2RScenario(scenario)) {
2324                         if (os != 'Windows_NT') {
2325                             return
2326                         }
2327                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2328                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2329                             return
2330                         }
2331                     }
2332                     else {
2333                         // Skip scenarios
2334                         switch (scenario) {
2335                             case 'ilrt':
2336                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2337                                 // the default scenario build
2338                                 if (os != 'Windows_NT') {
2339                                     return
2340                                 }
2341                                 // Only x64 for now
2342                                 if (architecture != 'x64') {
2343                                     return
2344                                 }
2345                                 // Release only
2346                                 if (configuration != 'Release') {
2347                                     return
2348                                 }
2349                                 break
2350                             case 'jitdiff':
2351                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2352                                     return
2353                                 }
2354                                 if (architecture != 'x64') {
2355                                     return
2356                                 }
2357                                 if (configuration != 'Checked') {
2358                                     return
2359                                 }
2360                                 break
2361                             case 'longgc':
2362                             case 'gcsimulator':
2363                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2364                                     return
2365                                 }
2366                                 if (architecture != 'x64') {
2367                                     return
2368                                 }
2369                                 if (configuration != 'Release') {
2370                                     return
2371                                 }
2372                                 break
2373                             case 'gc_reliability_framework':
2374                             case 'standalone_gc':
2375                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2376                                     return
2377                                 }
2378
2379                                 if (architecture != 'x64') {
2380                                     return
2381                                 }
2382
2383                                 if (configuration != 'Release' && configuration != 'Checked') {
2384                                     return
2385                                 }
2386                                 break
2387                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2388                             case 'formatting':
2389                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2390                                     return
2391                                 }
2392                                 if (architecture != 'x64') {
2393                                     return
2394                                 }
2395                                 if (configuration != 'Checked') {
2396                                     return
2397                                 }
2398                                 if (isBuildOnly) {
2399                                     return
2400                                 }
2401                                 break
2402                             case 'illink':
2403                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2404                                     return
2405                                 }
2406                                 if (architecture != 'x64' && architecture != 'x86') {
2407                                     return
2408                                 }
2409                                 if (isBuildOnly) {
2410                                     return
2411                                 }
2412                                 break
2413                             case 'normal':
2414                                 // Nothing skipped
2415                                 break
2416                             case 'innerloop':
2417                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2418                                     return
2419                                 }
2420                                 break
2421                             default:
2422                                 println("Unknown scenario: ${scenario}")
2423                                 assert false
2424                                 break
2425                         }
2426                     }
2427
2428                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2429                     // runtime behavior, not compile-time behavior, are not interesting.
2430                     switch (architecture) {
2431                         case 'x86_arm_altjit':
2432                         case 'x64_arm64_altjit':
2433                             if (isGCStressRelatedTesting(scenario)) {
2434                                 return
2435                             }
2436                             break
2437                         default:
2438                             break
2439                     }
2440
2441                     // Calculate names
2442                     def lowerConfiguration = configuration.toLowerCase()
2443                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2444                     def folderName = getJobFolder(scenario)
2445
2446                     // Create the new job
2447                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2448                     addToViews(newJob, isPR, architecture, os)
2449
2450                     def machineAffinityOptions = null
2451
2452                     if (os != 'Windows_NT') {
2453                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2454                     }
2455                     else {
2456                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2457                     }
2458
2459                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2460
2461                     // Add all the standard options
2462                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2463                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2464
2465                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2466                     def osGroup = getOSGroup(os)
2467
2468                     newJob.with {
2469                         steps {
2470                             if (os == 'Windows_NT') {
2471                                 buildCommands.each { buildCommand ->
2472                                     batchFile(buildCommand)
2473                                 }
2474                             }
2475                             else {
2476                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2477                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2478                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2479                                     // Define the Windows Tests and Corefx build job names
2480                                     def WindowsTestsName = projectFolder + '/' +
2481                                                            Utilities.getFullJobName(project,
2482                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2483                                                                                     false)
2484                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2485                                                        Utilities.getFolderName(branch)
2486
2487                                     // Copy the Windows test binaries and the Corefx build binaries
2488                                     copyArtifacts(WindowsTestsName) {
2489                                         includePatterns('bin/tests/tests.zip')
2490                                         buildSelector {
2491                                             latestSuccessful(true)
2492                                         }
2493                                     }
2494
2495                                     def arm_abi = 'arm'
2496                                     def corefx_os = 'linux'
2497                                     if (os == 'Tizen') {
2498                                         arm_abi = 'armel'
2499                                         corefx_os = 'tizen'
2500                                     }
2501
2502                                     // Let's use release CoreFX to test checked CoreCLR,
2503                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2504                                     def corefx_lowerConfiguration = lowerConfiguration
2505                                     if ( lowerConfiguration == 'checked' ) {
2506                                         corefx_lowerConfiguration='release'
2507                                     }
2508
2509                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2510                                         includePatterns('bin/build.tar.gz')
2511                                         buildSelector {
2512                                             latestSuccessful(true)
2513                                         }
2514                                     }
2515                                 }
2516
2517                                 buildCommands.each { buildCommand ->
2518                                     shell(buildCommand)
2519                                 }
2520                             }
2521                         }
2522                     } // newJob.with
2523
2524                 } // os
2525             } // configuration
2526         } // architecture
2527     } // isPR
2528 } // scenario
2529
2530
2531 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2532 Constants.allScenarios.each { scenario ->
2533     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2534
2535     [true, false].each { isPR ->
2536         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2537             Constants.crossList.each { os ->
2538                 if (architecture == 'arm64') {
2539                     if (os != "Ubuntu" && os != "Windows_NT") {
2540                         return
2541                     }
2542                 } else if (architecture == 'arm' || architecture == 'armlb') {
2543                     if (os != 'Windows_NT') {
2544                         return
2545                     }
2546                 } else if (architecture == 'x86') {
2547                     if (os != "Ubuntu") {
2548                         return
2549                     }
2550                 }
2551
2552                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2553
2554                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2555                     return
2556                 }
2557
2558                 Constants.configurationList.each { configuration ->
2559
2560                     // First, filter based on OS.
2561
2562                     if (os == 'Windows_NT') {
2563                         if (!isArmWindowsScenario(scenario)) {
2564                             return
2565                         }
2566                     }
2567                     else {
2568                         // Non-Windows
2569                         if (architecture == 'arm64') {
2570                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2571                                 return
2572                             }
2573                         }
2574                         else if (architecture == 'x86') {
2575                             // Linux/x86 only want innerloop and default test
2576                             if (!isNormalOrInnerloop) {
2577                                 return
2578                             }
2579                         }
2580                     }
2581
2582                     // For CentOS, we only want Checked/Release builds.
2583                     if (os == 'CentOS7.1') {
2584                         if (configuration != 'Checked' && configuration != 'Release') {
2585                             return
2586                         }
2587                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2588                             return
2589                         }
2590                     }
2591
2592                     // For RedHat and Debian, we only do Release builds.
2593                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2594                         if (configuration != 'Release') {
2595                             return
2596                         }
2597                         if (!isNormalOrInnerloop) {
2598                             return
2599                         }
2600                     }
2601
2602                     // Next, filter based on scenario.
2603
2604                     if (isJitStressScenario(scenario)) {
2605                         if (configuration != 'Checked') {
2606                             return
2607                         }
2608
2609                         // CoreFx JIT stress tests currently only implemented for ARM.
2610                         if (isCoreFxScenario(scenario) && (architecture != 'arm')) {
2611                             return
2612                         }
2613                     }
2614                     else if (isR2RBaselineScenario(scenario)) {
2615                         if (configuration != 'Checked' && configuration != 'Release') {
2616                             return
2617                         }
2618                     }
2619                     else if (isR2RStressScenario(scenario)) {
2620                         if (configuration != 'Checked') {
2621                             return
2622                         }
2623                     }
2624                     else {
2625                         // Skip scenarios
2626                         switch (scenario) {
2627                             case 'ilrt':
2628                             case 'longgc':
2629                             case 'gcsimulator':
2630                                 // Long GC tests take a long time on non-Release builds
2631                                 // ilrt is also Release only
2632                                 if (configuration != 'Release') {
2633                                     return
2634                                 }
2635                                 break
2636
2637                             case 'jitdiff':
2638                                 if (configuration != 'Checked') {
2639                                     return;
2640                                 }
2641                                 break
2642
2643                             case 'gc_reliability_framework':
2644                             case 'standalone_gc':
2645                                 if (configuration != 'Release' && configuration != 'Checked') {
2646                                     return
2647                                 }
2648                                 break
2649
2650                             case 'formatting':
2651                                 return
2652                             case 'illink':
2653                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2654                                     return
2655                                 }
2656                                 break
2657
2658                             case 'normal':
2659                                 // Nothing skipped
2660                                 break
2661
2662                             case 'innerloop':
2663                                 // Nothing skipped
2664                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2665                                     return
2666                                 }
2667                                 break
2668
2669                             default:
2670                                 println("Unknown scenario: ${scenario}")
2671                                 assert false
2672                                 break
2673                         }
2674                     }
2675
2676                     // Done filtering. Now, create the jobs.
2677
2678                     // =============================================================================================
2679                     // Create the test job
2680                     // =============================================================================================
2681
2682                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2683
2684                     def lowerConfiguration = configuration.toLowerCase()
2685                     def osGroup = getOSGroup(os)
2686                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2687
2688                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2689                     def inputCoreCLRBuildIsBuildOnly = false
2690                     if (isCoreFxScenario(scenario)) {
2691                         // Every CoreFx test depends on its own unique build.
2692                         inputCoreCLRBuildScenario = scenario
2693                         inputCoreCLRBuildIsBuildOnly = true
2694                     }
2695                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
2696                     def inputCoreCLRBuildName = projectFolder + '/' +
2697                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
2698
2699                     def inputWindowsTestsBuildName = ""
2700                     if (windowsArmJob != true) {
2701                         // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2702                         // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2703
2704                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2705
2706                         def inputWindowsTestBuildArch = architecture
2707                         if (architecture == "arm64" && os != "Windows_NT") {
2708                             // Use the x64 test build for arm64 unix
2709                             inputWindowsTestBuildArch = "x64"
2710                         }
2711
2712                         if (isJitStressScenario(scenario)) {
2713                             inputWindowsTestsBuildName = projectFolder + '/' +
2714                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2715                         } else {
2716                             inputWindowsTestsBuildName = projectFolder + '/' +
2717                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2718                         }
2719                     } // if (windowsArmJob != true)
2720
2721                     def serverGCString = ''
2722                     def testOpts = ''
2723
2724                     if (windowsArmJob != true) {
2725                         // Enable Server GC for Ubuntu PR builds
2726                         if (os == 'Ubuntu' && isPR) {
2727                             serverGCString = '--useServerGC'
2728                         }
2729
2730                         if (isR2RScenario(scenario)) {
2731
2732                             testOpts += ' --crossgen --runcrossgentests'
2733
2734                             if (scenario == 'r2r_jitstress1') {
2735                                 testOpts += ' --jitstress=1'
2736                             }
2737                             else if (scenario == 'r2r_jitstress2') {
2738                                 testOpts += ' --jitstress=2'
2739                             }
2740                             else if (scenario == 'r2r_jitstressregs1') {
2741                                 testOpts += ' --jitstressregs=1'
2742                             }
2743                             else if (scenario == 'r2r_jitstressregs2') {
2744                                 testOpts += ' --jitstressregs=2'
2745                             }
2746                             else if (scenario == 'r2r_jitstressregs3') {
2747                                 testOpts += ' --jitstressregs=3'
2748                             }
2749                             else if (scenario == 'r2r_jitstressregs4') {
2750                                 testOpts += ' --jitstressregs=4'
2751                             }
2752                             else if (scenario == 'r2r_jitstressregs8') {
2753                                 testOpts += ' --jitstressregs=8'
2754                             }
2755                             else if (scenario == 'r2r_jitstressregs0x10') {
2756                                 testOpts += ' --jitstressregs=0x10'
2757                             }
2758                             else if (scenario == 'r2r_jitstressregs0x80') {
2759                                 testOpts += ' --jitstressregs=0x80'
2760                             }
2761                             else if (scenario == 'r2r_jitstressregs0x1000') {
2762                                 testOpts += ' --jitstressregs=0x1000'
2763                             }
2764                             else if (scenario == 'r2r_jitminopts') {
2765                                 testOpts += ' --jitminopts'
2766                             }
2767                             else if (scenario == 'r2r_jitforcerelocs') {
2768                                 testOpts += ' --jitforcerelocs'
2769                             }
2770                             else if (scenario == 'r2r_gcstress15') {
2771                                 testOpts += ' --gcstresslevel=0xF'
2772                             }
2773                         }
2774                         else if (scenario == 'jitdiff') {
2775                             testOpts += ' --jitdisasm --crossgen'
2776                         }
2777                         else if (scenario == 'illink') {
2778                             testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2779                         }
2780                         else if (isLongGc(scenario)) {
2781                             // Long GC tests behave very poorly when they are not
2782                             // the only test running (many of them allocate until OOM).
2783                             testOpts += ' --sequential'
2784
2785                             // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2786                             // for running long GC and GCSimulator tests, respectively. We don't use them
2787                             // here because using a playlist file produces much more readable output on the CI machines
2788                             // and reduces running time.
2789                             //
2790                             // The Long GC playlist contains all of the tests that are
2791                             // going to be run. The GCSimulator playlist contains all of
2792                             // the GC simulator tests.
2793                             if (scenario == 'longgc') {
2794                                 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2795                             }
2796                             else if (scenario == 'gcsimulator') {
2797                                 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2798                             }
2799                         }
2800                         else if (isGcReliabilityFramework(scenario)) {
2801                             testOpts += ' --build-overlay-only'
2802                         }
2803                         else if (scenario == 'standalone_gc') {
2804                             if (osGroup == 'OSX') {
2805                                 testOpts += ' --gcname=libclrgc.dylib'
2806                             }
2807                             else if (osGroup == 'Linux') {
2808                                 testOpts += ' --gcname=libclrgc.so'
2809                             }
2810                             else {
2811                                 println("Unexpected OS group: ${osGroup} for os ${os}")
2812                                 assert false
2813                             }
2814                         }
2815                     } // if (windowsArmJob != true)
2816
2817                     def folder = getJobFolder(scenario)
2818                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2819                         // Add parameters for the inputs
2820
2821                         if (windowsArmJob == true) {
2822                             parameters {
2823                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2824                             }
2825                         }
2826                         else {
2827                             parameters {
2828                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
2829                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2830                             }
2831                         }
2832
2833                         steps {
2834                             // Set up the copies
2835
2836                             // Coreclr build containing the tests and mscorlib
2837                             // pri1 jobs still need to copy windows_nt built tests
2838                             if (windowsArmJob != true) {
2839                                 copyArtifacts(inputWindowsTestsBuildName) {
2840                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2841                                     buildSelector {
2842                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2843                                     }
2844                                 }
2845                             }
2846
2847                             // Coreclr build we are trying to test
2848                             //
2849                             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2850
2851                             copyArtifacts(inputCoreCLRBuildName) {
2852                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2853                                 buildSelector {
2854                                     buildNumber('${CORECLR_BUILD}')
2855                                 }
2856                             }
2857
2858                             // Windows CoreCLR Arm(64) will restore corefx
2859                             // packages correctly.
2860                             //
2861                             // In addition, test steps are entirely different
2862                             // because we do not have a unified runner
2863                             if (windowsArmJob != true) {
2864                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2865
2866                                 // HACK -- Arm64 does not have corefx jobs yet.
2867                                 // Clone corefx and build the native packages overwriting the x64 packages.
2868                                 if (architecture == 'arm64') {
2869                                     shell("mkdir -p ./bin/CoreFxBinDir")
2870                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2871                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2872                                 }
2873                                 else if (architecture == 'x86') {
2874                                     shell("mkdir ./bin/CoreFxNative")
2875
2876                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2877                                         includePatterns('bin/build.tar.gz')
2878                                         targetDirectory('bin/CoreFxNative')
2879                                         buildSelector {
2880                                             latestSuccessful(true)
2881                                         }
2882                                     }
2883
2884                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2885                                 }
2886
2887                                 // Unzip the tests first.  Exit with 0
2888                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2889                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2890
2891                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2892
2893                                 // Execute the tests
2894                                 def runDocker = isNeedDocker(architecture, os, false)
2895                                 def dockerPrefix = ""
2896                                 def dockerCmd = ""
2897                                 if (runDocker) {
2898                                     def dockerImage = getDockerImageName(architecture, os, false)
2899                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2900                                     dockerCmd = dockerPrefix + "${dockerImage} "
2901                                 }
2902
2903                                 // If we are running a stress mode, we'll set those variables first
2904                                 def testEnvOpt = ""
2905                                 if (isJitStressScenario(scenario)) {
2906                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2907                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2908                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2909                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2910                                     shell("${envScriptCmds}")
2911                                     testEnvOpt = "--test-env=" + scriptFileName
2912                                 }
2913
2914                                 if (isGCStressRelatedTesting(scenario)) {
2915                                     shell('./init-tools.sh')
2916                                 }
2917
2918                                 shell("""${dockerCmd}./tests/runtest.sh \\
2919                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2920                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2921                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2922                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2923
2924                                 if (isGcReliabilityFramework(scenario)) {
2925                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2926                                     if (serverGCString != '') {
2927                                         if (runDocker) {
2928                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2929                                         }
2930                                         else {
2931                                             shell("export COMPlus_gcServer=1")
2932                                         }
2933                                     }
2934
2935                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2936                                 }
2937                             }
2938                             else { // windowsArmJob == true
2939
2940                                 if (isCoreFxScenario(scenario)) {
2941
2942                                     // Only arm supported for corefx testing now.
2943                                     assert architecture == 'arm'
2944
2945                                     // Unzip CoreFx runtime
2946                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2947
2948                                     // Unzip CoreFx tests.
2949                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2950
2951                                     // Add the script to run the corefx tests
2952                                     def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2953                                     def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2954                                     def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2955                                     batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2956
2957                                 } else { // !isCoreFxScenario(scenario)
2958
2959                                     // Unzip tests.
2960                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2961
2962                                     def buildCommands = ""
2963
2964                                     def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2965                                     def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2966                                     def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2967
2968                                     // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2969                                     addCommand("SETLOCAL ENABLEEXTENSIONS")
2970
2971                                     // For all jobs
2972                                     addEnvVariable("CORE_ROOT", coreRootLocation)
2973
2974                                     addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2975                                     addEnvVariable("COMPlus_ContinueOnAssert", "0")
2976
2977                                     // ARM legacy backend; this is an altjit.
2978                                     if (architecture == "armlb") {
2979                                         addEnvVariable("COMPlus_AltJit", "*")
2980                                         addEnvVariable("COMPlus_AltJitNgen", "*")
2981                                         addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2982                                         addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2983                                     }
2984
2985                                     // If we are running a stress mode, we'll set those variables as well
2986                                     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2987                                         def stressValues = null
2988                                         if (isJitStressScenario(scenario)) {
2989                                             stressValues = Constants.jitStressModeScenarios[scenario]
2990                                         }
2991                                         else {
2992                                             stressValues = Constants.r2rStressScenarios[scenario]
2993                                         }
2994
2995                                         stressValues.each { key, value ->
2996                                             addEnvVariable(key, value)
2997                                         }
2998                                     }
2999
3000                                     if (isR2RScenario(scenario)) {
3001                                         // Crossgen the framework assemblies.
3002                                         buildCommands += """
3003 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3004 @goto skip_PrecompileAssembly
3005
3006 :PrecompileAssembly
3007 @REM Skip mscorlib since it is already precompiled.
3008 @if /I "%3" == "mscorlib.dll" exit /b 0
3009 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3010
3011 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3012 @if "%errorlevel%" == "-2146230517" (
3013     echo %2 is not a managed assembly.
3014 ) else if "%errorlevel%" == "-2146234344" (
3015     echo %2 is not a managed assembly.
3016 ) else if %errorlevel% neq 0 (
3017     echo Unable to precompile %2
3018 ) else (
3019     echo Precompiled %2
3020 )
3021 @exit /b 0
3022
3023 :skip_PrecompileAssembly
3024 """
3025
3026                                         // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3027                                         // crossgen on tests before running them.
3028                                         addEnvVariable("RunCrossGen", "true")
3029                                     } // isR2RScenario(scenario)
3030
3031                                     // Create the smarty command
3032                                     def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3033                                     def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3034                                     def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3035
3036                                     def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
3037
3038                                     if (architecture == "armlb") {
3039                                         addExclude("LEGACYJIT_FAIL")
3040                                     }
3041
3042                                     // Exclude tests based on scenario.
3043                                     Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3044                                         addArchSpecificExclude(architecture, excludeTag)
3045                                     }
3046
3047                                     // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
3048                                     if (scenario == 'innerloop') {
3049                                         addExclude("pri1")
3050                                     }
3051
3052                                     // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3053                                     // TODO: We should create a "long running" job that runs these with a longer timeout.
3054                                     addExclude("LONG_RUNNING")
3055
3056                                     smartyCommand += "/lstFile Tests.lst"
3057
3058                                     def testListArch = [
3059                                         'arm64': 'arm64',
3060                                         'arm': 'arm',
3061                                         'armlb': 'arm'
3062                                     ]
3063
3064                                     def archLocation = testListArch[architecture]
3065
3066                                     addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3067                                     addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3068                                     addCommand("${smartyCommand}")
3069
3070                                     // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3071                                     // However, we also need to remove all the variables that were set during this batch file, so we
3072                                     // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3073                                     // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3074                                     // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3075                                     // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3076                                     // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3077                                     addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3078
3079                                     // ZIP up the smarty output, no matter what the smarty result.
3080                                     addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3081
3082                                     addCommand("echo %errorlevel%")
3083                                     addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3084
3085                                     // Use the smarty errorlevel as the script errorlevel.
3086                                     addCommand("exit /b %__save_smarty_errorlevel%")
3087
3088                                     batchFile(buildCommands)
3089                                 } // non-corefx testing
3090                             } // windowsArmJob == true
3091                         } // steps
3092                     } // job
3093
3094                     addToViews(newJob, isPR, architecture, os)
3095
3096                     if (scenario == 'jitdiff') {
3097                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3098                     }
3099
3100                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3101                     if (os in ['Ubuntu']) {
3102                         SummaryBuilder summaries = new SummaryBuilder()
3103                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3104                         summaries.emit(newJob)
3105                     }
3106
3107                     def affinityOptions = null
3108
3109                     if (windowsArmJob == true) {
3110                         affinityOptions = [
3111                             "use_arm64_build_machine" : false
3112                         ]
3113                     }
3114
3115                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3116                         affinityOptions = [
3117                             "large_pages" : false
3118                         ]
3119                     }
3120
3121                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3122                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3123
3124                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3125
3126                     if (windowsArmJob != true) {
3127                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3128                     }
3129                     else {
3130                         if (!isCoreFxScenario(scenario)) {
3131                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3132
3133                             // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3134                             // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3135                             // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3136                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3137                         }
3138                     }
3139
3140                     // =============================================================================================
3141                     // Create a build flow to join together the build and tests required to run this test.
3142                     // =============================================================================================
3143
3144                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3145                     // Linux CoreCLR test
3146                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3147                     def fullTestJobName = projectFolder + '/' + newJob.name
3148                     // Add a reference to the input jobs for report purposes
3149                     JobReport.Report.addReference(inputCoreCLRBuildName)
3150                     if (windowsArmJob != true) {
3151                         JobReport.Report.addReference(inputWindowsTestsBuildName)
3152                     }
3153                     JobReport.Report.addReference(fullTestJobName)
3154                     def newFlowJob = null
3155
3156                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3157                         // Do not create the flow job for RHEL jobs.
3158                         return
3159                     }
3160
3161                     // For pri0 jobs we can build tests on unix
3162                     if (windowsArmJob) {
3163                         // For Windows arm jobs there is no reason to build a parallel test job.
3164                         // The product build supports building and archiving the tests.
3165
3166                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3167                         buildFlow("""\
3168 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3169
3170 // And then build the test build
3171 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3172 """)
3173                         }
3174                     }
3175                     else {
3176                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3177                         buildFlow("""\
3178 // Build the input jobs in parallel
3179 parallel (
3180 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3181 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3182 )
3183
3184 // And then build the test build
3185 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3186             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3187 """)
3188                         }
3189                     }
3190
3191                     addToViews(newFlowJob, isPR, architecture, os)
3192
3193                     // For the flow jobs set the machine affinity as x64 if an armarch.
3194                     def flowArch = architecture
3195
3196                     if (flowArch in validWindowsNTCrossArches) {
3197                         flowArch = 'x64'
3198                         affinityOptions = null
3199                     }
3200
3201                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3202                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3203                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3204                 } // configuration
3205             } // os
3206         } // architecture
3207     } // isPR
3208 } // scenario
3209
3210 JobReport.Report.generateJobReport(out)
3211
3212 // Make the call to generate the help job
3213 Utilities.createHelperJob(this, project, branch,
3214     "Welcome to the ${project} Repository",  // This is prepended to the help message
3215     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3216
3217 Utilities.addCROSSCheck(this, project, branch)