netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'jitincompletehwintrinsic'
 266                // 'jitx86hwintrinsicnoavx'
 267                // 'jitx86hwintrinsicnoavx2'
 268                // 'jitx86hwintrinsicnosimd'
 269                // 'jitnox86hwintrinsic'
 270                'corefx_baseline':                        [],    // corefx tests don't use smarty
 271                'corefx_minopts':                         [],    // corefx tests don't use smarty
 272                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 273                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 274                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 275                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 276                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 277                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 278                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 279                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 280                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 281                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 283                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 284                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 285                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 286                'heapverify1':                            [],
 287                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 288                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 290                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 293
 294                //
 295                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 296                //
 297
 298                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 299                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 317     ]
 318
 319     def static validLinuxArm64Scenarios = [
 320         'normal',
 321         'r2r',
 322         'innerloop',
 323         'gcstress0x3',
 324         'gcstress0xc'
 325     ]
 326
 327     def static configurationList = ['Debug', 'Checked', 'Release']
 328
 329     // This is the set of architectures
 330     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 331 }
 332
 333 // **************************************************************
 334 // Create some specific views
 335 //
 336 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 337 // views based on a single regular expression. These views will be generated by adding a
 338 // specific set of jobs to them.
 339 //
 340 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 341 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 342 // those are required, don't add them (which simplifies the view pages, as well).
 343 // **************************************************************
 344
 345 class Views {
 346     def static MergeJobView = null
 347     def static PeriodicJobView = null
 348     def static ArchitectureViews = [:]
 349     def static OSViews = [:]
 350 }
 351
 352 // MergeJobView: include all jobs that execute when a PR change is merged.
 353 Views.MergeJobView = listView('Merge') {
 354     recurse()
 355     columns {
 356         status()
 357         weather()
 358         name()
 359         lastSuccess()
 360         lastFailure()
 361         lastDuration()
 362         buildButton()
 363     }
 364 }
 365
 366 // PeriodicJobView: include all jobs that execute on a schedule
 367 Views.PeriodicJobView = listView('Periodic') {
 368     recurse()
 369     columns {
 370         status()
 371         weather()
 372         name()
 373         lastSuccess()
 374         lastFailure()
 375         lastDuration()
 376         buildButton()
 377     }
 378 }
 379
 380 // Create a view for non-PR jobs for each architecture.
 381 Constants.architectureList.each { architecture ->
 382     Views.ArchitectureViews[architecture] = listView(architecture) {
 383         recurse()
 384         columns {
 385             status()
 386             weather()
 387             name()
 388             lastSuccess()
 389             lastFailure()
 390             lastDuration()
 391             buildButton()
 392         }
 393     }
 394 }
 395
 396 // Create a view for non-PR jobs for each OS.
 397 Constants.osList.each { os ->
 398     // Don't create one for the special 'Windows_NT_BuildOnly'
 399     if (os == 'Windows_NT_BuildOnly') {
 400         return
 401     }
 402     Views.OSViews[os] = listView(os) {
 403         recurse()
 404         columns {
 405             status()
 406             weather()
 407             name()
 408             lastSuccess()
 409             lastFailure()
 410             lastDuration()
 411             buildButton()
 412         }
 413     }
 414 }
 415
 416 def static addToMergeView(def job) {
 417     Views.MergeJobView.with {
 418         jobs {
 419             name(job.name)
 420         }
 421     }
 422 }
 423
 424 def static addToPeriodicView(def job) {
 425     Views.PeriodicJobView.with {
 426         jobs {
 427             name(job.name)
 428         }
 429     }
 430 }
 431
 432 def static addToViews(def job, def isPR, def architecture, def os) {
 433     if (isPR) {
 434         // No views want PR jobs currently.
 435         return
 436     }
 437
 438     // Add to architecture view.
 439     Views.ArchitectureViews[architecture].with {
 440         jobs {
 441             name(job.name)
 442         }
 443     }
 444
 445     // Add to OS view.
 446     Views.OSViews[os].with {
 447         jobs {
 448             name(job.name)
 449         }
 450     }
 451 }
 452
 453 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 454     addToPeriodicView(job)
 455     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 456 }
 457
 458 def static addGithubPushTriggerHelper(def job) {
 459     addToMergeView(job)
 460     Utilities.addGithubPushTrigger(job)
 461 }
 462
 463
 464 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 465     assert os instanceof String
 466     assert architecture instanceof String
 467
 468     def armArches = ['arm', 'armlb', 'arm64']
 469     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 470
 471     if (!(architecture in armArches)) {
 472         assert options == null
 473         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 474
 475         return
 476     }
 477
 478     // This is an arm(64) job.
 479     //
 480     // There are several options.
 481     //
 482     // Windows_NT
 483     //
 484     // Arm32 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 486     // Arm32 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 488     //
 489     // Arm64 (Build) -> latest-arm64
 490     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 491     // Arm64 (Test)  -> arm64-windows_nt
 492     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 493     //
 494     // Ubuntu
 495     //
 496     // Arm32 (Build) -> arm-cross-latest
 497     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 498     // Arm32 (Test)  -> NYI Arch not supported
 499     //       |->
 500     //
 501     // Arm64 (Build) -> arm64-cross-latest
 502     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 503     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 504     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 505     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 506     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 507
 508     // This has to be a arm arch
 509     assert architecture in armArches
 510     if (os == "Windows_NT") {
 511         // Arm(64) Windows jobs share the same machines for now
 512         def isBuild = options['use_arm64_build_machine'] == true
 513
 514         if (isBuild == true) {
 515             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 516         } else {
 517             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 518         }
 519     } else {
 520         assert os != 'Windows_NT'
 521         assert os in supportedArmLinuxOs
 522
 523         if (architecture == 'arm' || architecture == 'armlb') {
 524             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 525         } else {
 526             // Arm64 Linux
 527             if (options['is_build_only'] == true) {
 528                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 529             } else {
 530                 // Arm64 Test Machines
 531                 if (options['large_pages'] == false) {
 532                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 533                 } else {
 534                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 535                 }
 536             }
 537         }
 538     }
 539 }
 540
 541 def static isGCStressRelatedTesting(def scenario) {
 542     // The 'r2r_gcstress15' scenario is a basic scenario.
 543     // Detect it and make it a GCStress related.
 544     if (scenario == 'r2r_gcstress15')
 545     {
 546         return true;
 547     }
 548
 549     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 550     def scenarioName = scenario.toLowerCase()
 551     def isGCStressTesting = false
 552     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 553         if (k in gcStressTestEnvVars) {
 554             isGCStressTesting = true;
 555         }
 556     }
 557     return isGCStressTesting
 558 }
 559
 560 def static isCoreFxScenario(def scenario) {
 561     def corefx_prefix = 'corefx_'
 562     if (scenario.length() < corefx_prefix.length()) {
 563         return false
 564     }
 565     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 566 }
 567
 568 def static isR2RBaselineScenario(def scenario) {
 569     return (scenario == 'r2r')
 570 }
 571
 572 def static isR2RStressScenario(def scenario) {
 573     return Constants.r2rStressScenarios.containsKey(scenario)
 574 }
 575
 576 def static isR2RScenario(def scenario) {
 577     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 578 }
 579
 580 def static isJitStressScenario(def scenario) {
 581     return Constants.jitStressModeScenarios.containsKey(scenario)
 582 }
 583
 584 def static isLongGc(def scenario) {
 585     return (scenario == 'longgc' || scenario == 'gcsimulator')
 586 }
 587
 588 def static isJitDiff(def scenario) {
 589     return (scenario == 'jitdiff')
 590 }
 591
 592 def static isGcReliabilityFramework(def scenario) {
 593     return (scenario == 'gc_reliability_framework')
 594 }
 595
 596 def static isArmWindowsScenario(def scenario) {
 597     return Constants.validArmWindowsScenarios.containsKey(scenario)
 598 }
 599
 600 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 601     if (isBuildOnly == true) {
 602         os = 'Windows_NT_buildOnly'
 603     }
 604
 605     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 606
 607     if (validOsPrTriggerArchConfigs == null) {
 608         return false
 609     }
 610
 611     if (validOsPrTriggerArchConfigs[architecture] != null) {
 612         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 613
 614         if (!(configuration in validOsPrTriggerConfigs)) {
 615             return false
 616         }
 617     } else {
 618         return false
 619     }
 620
 621     return true
 622 }
 623
 624 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 625     // 2 hours (120 minutes) is the default timeout
 626     def timeout = 120
 627     def innerLoop = (scenario == "innerloop")
 628
 629     if (!innerLoop) {
 630         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 631         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 632         timeout = 240
 633     }
 634
 635     if (!isBuildOnly) {
 636         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 637         if (isGCStressRelatedTesting(scenario)) {
 638             timeout = 4320
 639         }
 640         else if (isCoreFxScenario(scenario)) {
 641             timeout = 360
 642         }
 643         else if (isJitStressScenario(scenario)) {
 644             timeout = 300
 645         }
 646         else if (isR2RBaselineScenario(scenario)) {
 647             timeout = 240
 648         }
 649         else if (isLongGc(scenario)) {
 650             timeout = 1440
 651         }
 652         else if (isJitDiff(scenario)) {
 653             timeout = 240
 654         }
 655         else if (isGcReliabilityFramework(scenario)) {
 656             timeout = 1440
 657         }
 658         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 659             timeout = 240
 660         }
 661     }
 662
 663     if (configuration == 'Debug') {
 664         // Debug runs can be very slow. Add an hour.
 665         timeout += 60
 666     }
 667
 668     // If we've changed the timeout from the default, set it in the job.
 669
 670     if (timeout != 120) {
 671         Utilities.setJobTimeout(newJob, timeout)
 672     }
 673 }
 674
 675 def static getJobFolder(def scenario) {
 676     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 677         return 'jitstress'
 678     }
 679     if (scenario == 'illink') {
 680         return 'illink'
 681     }
 682     return ''
 683 }
 684
 685 def static getStressModeDisplayName(def scenario) {
 686     def displayStr = ''
 687     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 688         def prefixLength = 'COMPlus_'.length()
 689         if (k.length() >= prefixLength) {
 690             def modeName = k.substring(prefixLength, k.length())
 691             displayStr += ' ' + modeName + '=' + v
 692         }
 693     }
 694
 695     if (isCoreFxScenario(scenario)) {
 696         displayStr = ('CoreFx ' + displayStr).trim()
 697     }
 698
 699     return displayStr
 700 }
 701
 702 def static getR2RDisplayName(def scenario) {
 703     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 704     def displayStr = scenario
 705     def prefixLength = 'r2r_'.length()
 706     if (displayStr.length() >= prefixLength) {
 707         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 708     } else if (scenario == 'r2r') {
 709         displayStr = "R2R"
 710     }
 711     return displayStr
 712 }
 713
 714 //
 715 // Functions to create an environment script.
 716 //      envScriptCreate -- initialize the script (call first)
 717 //      envScriptFinalize -- finalize the script (call last)
 718 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 719 //      envScriptAppendExistingScript -- append an existing script to the generated script
 720 //
 721 // Each script returns a string of commands. Concatenate all the strings together before
 722 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 723 //
 724
 725 // Initialize the environment setting script.
 726 def static envScriptCreate(def os, def stepScriptLocation) {
 727     def stepScript = ''
 728     if (os == 'Windows_NT') {
 729         stepScript += "echo Creating TestEnv script\r\n"
 730         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 731
 732         // Create at least an empty script.
 733         stepScript += "echo. > ${stepScriptLocation}\r\n"
 734     }
 735     else {
 736         stepScript += "echo Creating environment setting script\n"
 737         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 738     }
 739
 740     return stepScript
 741 }
 742
 743 // Generates the string for setting stress mode variables.
 744 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 745     def stepScript = ''
 746     if (os == 'Windows_NT') {
 747         stressModeVars.each{ k, v ->
 748             // Write out what we are writing to the script file
 749             stepScript += "echo Setting ${k}=${v}\r\n"
 750             // Write out the set itself to the script file`
 751             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 752         }
 753     }
 754     else {
 755         stressModeVars.each{ k, v ->
 756             // Write out what we are writing to the script file
 757             stepScript += "echo Setting ${k}=${v}\n"
 758             // Write out the set itself to the script file`
 759             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 760         }
 761     }
 762
 763     return stepScript
 764 }
 765
 766 // Append an existing script to an environment script.
 767 // Returns string of commands to do this.
 768 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 769     assert (os == 'Windows_NT')
 770     def stepScript = ''
 771
 772     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 773     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 774
 775     return stepScript
 776 }
 777
 778 // Finalize an environment setting script.
 779 // Returns string of commands to do this.
 780 def static envScriptFinalize(def os, def stepScriptLocation) {
 781     def stepScript = ''
 782
 783     if (os == 'Windows_NT') {
 784         // Display the resulting script. This is useful when looking at the output log file.
 785         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 786         stepScript += "type ${stepScriptLocation}\r\n"
 787     }
 788     else {
 789         stepScript += "chmod +x ${stepScriptLocation}\n"
 790     }
 791
 792     return stepScript
 793 }
 794
 795 def static isNeedDocker(def architecture, def os, def isBuild) {
 796     if (isBuild) {
 797         if (architecture == 'x86' && os == 'Ubuntu') {
 798             return true
 799         }
 800         else if (architecture == 'arm') {
 801             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 802                 return true
 803             }
 804         }
 805     }
 806     else {
 807         if (architecture == 'x86' && os == 'Ubuntu') {
 808             return true
 809         }
 810     }
 811     return false
 812 }
 813
 814 def static getDockerImageName(def architecture, def os, def isBuild) {
 815     // We must change some docker private images to official later
 816     if (isBuild) {
 817         if (architecture == 'x86' && os == 'Ubuntu') {
 818             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 819         }
 820         else if (architecture == 'arm') {
 821             if (os == 'Ubuntu') {
 822                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 823             }
 824             else if (os == 'Ubuntu16.04') {
 825                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 826             }
 827             else if (os == 'Tizen') {
 828                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 829             }
 830         }
 831     }
 832     else {
 833         if (architecture == 'x86' && os == 'Ubuntu') {
 834             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 835         }
 836     }
 837     println("Unknown architecture to use docker: ${architecture} ${os}");
 838     assert false
 839 }
 840
 841 // Calculates the name of the build job based on some typical parameters.
 842 //
 843 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 844     // If the architecture is x64, do not add that info into the build name.
 845     // Need to change around some systems and other builds to pick up the right builds
 846     // to do that.
 847
 848     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 849     if (isBuildOnly) {
 850         suffix += '_bld'
 851     }
 852     def baseName = ''
 853     switch (architecture) {
 854         case 'x64':
 855             if (scenario == 'normal') {
 856                 // For now we leave x64 off of the name for compatibility with other jobs
 857                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 858             }
 859             else if (scenario == 'formatting') {
 860                 // we don't care about the configuration for the formatting job. It runs all configs
 861                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 862             }
 863             else {
 864                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 865             }
 866             break
 867         case 'arm64':
 868             if (os.toLowerCase() == "windows_nt") {
 869                 // These are cross builds
 870                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 871             }
 872             else {
 873                 // Defaults to a small page size set of machines.
 874                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 875             }
 876             break
 877         case 'arm':
 878             // These are cross builds
 879             if (os == 'Tizen') {
 880                 // ABI: softfp
 881                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 882             }
 883             else {
 884                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 885             }
 886             break
 887         case 'armlb':
 888             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 889             break
 890         case 'x86':
 891         case 'x86_arm_altjit':
 892         case 'x64_arm64_altjit':
 893             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 894             break
 895         default:
 896             println("Unknown architecture: ${architecture}");
 897             assert false
 898             break
 899     }
 900
 901     return baseName + suffix
 902 }
 903
 904 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 905
 906     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 907     if (os == 'Windows_NT') {
 908         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 909             if (!(branch in Constants.WindowsArm64Branches)) {
 910                 return
 911             }
 912         }
 913     }
 914
 915     // Check scenario.
 916     switch (scenario) {
 917         case 'innerloop':
 918             break
 919         case 'normal':
 920             switch (architecture) {
 921                 case 'x64':
 922                 case 'x86':
 923                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 924                         addPeriodicTriggerHelper(job, '@daily')
 925                     }
 926                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 927                         addGithubPushTriggerHelper(job)
 928                     }
 929                     break
 930                 case 'arm':
 931                 case 'armlb':
 932                 case 'x86_arm_altjit':
 933                 case 'x64_arm64_altjit':
 934                     addGithubPushTriggerHelper(job)
 935                     break
 936                 case 'arm64':
 937                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 938                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 939                     break
 940                 default:
 941                     println("Unknown architecture: ${architecture}");
 942                     assert false
 943                     break
 944             }
 945             break
 946         case 'r2r':
 947             assert !(os in bidailyCrossList)
 948             // r2r gets a push trigger for checked/release
 949             if (configuration == 'Checked' || configuration == 'Release') {
 950                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 951                 if (architecture == 'x64' && os != 'OSX10.12') {
 952                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 953                     if (isFlowJob || os == 'Windows_NT') {
 954                         addGithubPushTriggerHelper(job)
 955                     }
 956                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 957                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 958                     if (isFlowJob) {
 959                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 960                     }
 961                 }
 962                 // For x86, only add per-commit jobs for Windows
 963                 else if (architecture == 'x86') {
 964                     if (os == 'Windows_NT') {
 965                         addGithubPushTriggerHelper(job)
 966                     }
 967                 }
 968                 // arm64 r2r jobs should only run daily.
 969                 else if (architecture == 'arm64') {
 970                     if (os == 'Windows_NT') {
 971                         addPeriodicTriggerHelper(job, '@daily')
 972                     }
 973                 }
 974             }
 975             break
 976         case 'r2r_jitstress1':
 977         case 'r2r_jitstress2':
 978         case 'r2r_jitstressregs1':
 979         case 'r2r_jitstressregs2':
 980         case 'r2r_jitstressregs3':
 981         case 'r2r_jitstressregs4':
 982         case 'r2r_jitstressregs8':
 983         case 'r2r_jitstressregs0x10':
 984         case 'r2r_jitstressregs0x80':
 985         case 'r2r_jitstressregs0x1000':
 986         case 'r2r_jitminopts':
 987         case 'r2r_jitforcerelocs':
 988         case 'r2r_gcstress15':
 989             assert !(os in bidailyCrossList)
 990
 991             // GCStress=C is currently not supported on OS X
 992             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 993                 break
 994             }
 995
 996             // GC Stress 15 r2r gets a push trigger for checked/release
 997             if (configuration == 'Checked' || configuration == 'Release') {
 998                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 999                 if (architecture == 'x64') {
1000                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1001                     if (isFlowJob || os == 'Windows_NT') {
1002                         // Add a weekly periodic trigger
1003                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1004                     }
1005                 }
1006                 // For x86, only add per-commit jobs for Windows
1007                 else if (architecture == 'x86') {
1008                     if (os == 'Windows_NT') {
1009                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1010                     }
1011                 }
1012             }
1013             break
1014         case 'longgc':
1015             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1016             assert configuration == 'Release'
1017             assert architecture == 'x64'
1018             addPeriodicTriggerHelper(job, '@daily')
1019             // TODO: Add once external email sending is available again
1020             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1021             break
1022         case 'gcsimulator':
1023             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1024             assert configuration == 'Release'
1025             assert architecture == 'x64'
1026             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1027             // TODO: Add once external email sending is available again
1028             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1029             break
1030         case 'standalone_gc':
1031             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1032             assert (configuration == 'Release' || configuration == 'Checked')
1033             // TODO: Add once external email sending is available again
1034             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1035             addPeriodicTriggerHelper(job, '@daily')
1036             break
1037         case 'gc_reliability_framework':
1038             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1039             assert (configuration == 'Release' || configuration == 'Checked')
1040             // Only triggered by phrase.
1041             break
1042         case 'ilrt':
1043             assert !(os in bidailyCrossList)
1044             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1045             if (architecture == 'x64' && configuration == 'Release') {
1046                 // We don't expect to see a job generated except in these scenarios
1047                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1048                 if (isFlowJob || os == 'Windows_NT') {
1049                     addPeriodicTriggerHelper(job, '@daily')
1050                 }
1051             }
1052             break
1053         case 'jitdiff':
1054             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1055             assert configuration == 'Checked'
1056             assert (architecture == 'x64' || architecture == 'x86')
1057             addGithubPushTriggerHelper(job)
1058             break
1059         case 'formatting':
1060             assert (os == 'Windows_NT' || os == "Ubuntu")
1061             assert architecture == 'x64'
1062             addGithubPushTriggerHelper(job)
1063             break
1064         case 'jitstressregs1':
1065         case 'jitstressregs2':
1066         case 'jitstressregs3':
1067         case 'jitstressregs4':
1068         case 'jitstressregs8':
1069         case 'jitstressregs0x10':
1070         case 'jitstressregs0x80':
1071         case 'jitstressregs0x1000':
1072         case 'minopts':
1073         case 'forcerelocs':
1074         case 'jitstress1':
1075         case 'jitstress2':
1076         case 'jitstress2_jitstressregs1':
1077         case 'jitstress2_jitstressregs2':
1078         case 'jitstress2_jitstressregs3':
1079         case 'jitstress2_jitstressregs4':
1080         case 'jitstress2_jitstressregs8':
1081         case 'jitstress2_jitstressregs0x10':
1082         case 'jitstress2_jitstressregs0x80':
1083         case 'jitstress2_jitstressregs0x1000':
1084         case 'tailcallstress':
1085         case 'jitsse2only':
1086         case 'jitnosimd':
1087         case 'jitnox86hwintrinsic':
1088         case 'jitincompletehwintrinsic':
1089         case 'jitx86hwintrinsicnoavx':
1090         case 'jitx86hwintrinsicnoavx2':
1091         case 'jitx86hwintrinsicnosimd':
1092         case 'corefx_baseline':
1093         case 'corefx_minopts':
1094         case 'corefx_jitstress1':
1095         case 'corefx_jitstress2':
1096         case 'corefx_jitstressregs1':
1097         case 'corefx_jitstressregs2':
1098         case 'corefx_jitstressregs3':
1099         case 'corefx_jitstressregs4':
1100         case 'corefx_jitstressregs8':
1101         case 'corefx_jitstressregs0x10':
1102         case 'corefx_jitstressregs0x80':
1103         case 'corefx_jitstressregs0x1000':
1104         case 'zapdisable':
1105             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1106                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1107                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1108                     if (os == 'Windows_NT') {
1109                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1110                         addPeriodicTriggerHelper(job, '@weekly')
1111                     }
1112                 }
1113                 else {
1114                     addPeriodicTriggerHelper(job, '@daily')
1115                 }
1116             }
1117             break
1118         case 'heapverify1':
1119         case 'gcstress0x3':
1120             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1121                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1122                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1123                     if (os == 'Windows_NT') {
1124                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1125                         addPeriodicTriggerHelper(job, '@weekly')
1126                     }
1127                     // TODO: Add once external email sending is available again
1128                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1129                 }
1130                 else {
1131                     addPeriodicTriggerHelper(job, '@weekly')
1132                 }
1133             }
1134             break
1135         case 'gcstress0xc':
1136         case 'gcstress0xc_zapdisable':
1137         case 'gcstress0xc_zapdisable_jitstress2':
1138         case 'gcstress0xc_zapdisable_heapverify1':
1139         case 'gcstress0xc_jitstress1':
1140         case 'gcstress0xc_jitstress2':
1141         case 'gcstress0xc_minopts_heapverify1':
1142             // GCStress=C is currently not supported on OS X
1143             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1144                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1145                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1146                     if (os == 'Windows_NT') {
1147                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1148                         addPeriodicTriggerHelper(job, '@weekly')
1149                     }
1150                     // TODO: Add once external email sending is available again
1151                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1152                 }
1153                 else {
1154                     addPeriodicTriggerHelper(job, '@weekly')
1155                 }
1156             }
1157             break
1158
1159         case 'illink':
1160             // Testing on other operating systems TBD
1161             assert (os == 'Windows_NT' || os == 'Ubuntu')
1162             if (architecture == 'x64' || architecture == 'x86') {
1163                 if (configuration == 'Checked') {
1164                     addPeriodicTriggerHelper(job, '@daily')
1165                 }
1166             }
1167             break
1168
1169         case 'tieredcompilation':
1170         case 'corefx_tieredcompilation':
1171             // No periodic jobs just yet, still testing
1172             break
1173
1174         default:
1175             println("Unknown scenario: ${scenario}");
1176             assert false
1177             break
1178     }
1179     return
1180 }
1181
1182 // **************************
1183 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1184 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1185 // builds will do a couple extra steps.
1186 // **************************
1187
1188 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1189 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1190 // it's just used for internal builds
1191 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1192 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1193     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1194
1195     if (isWindowsBuildOnlyJob) {
1196         return
1197     }
1198
1199     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1200     // Non pull request builds.
1201     if (!isPR) {
1202         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1203         return
1204     }
1205
1206      def arm64Users = [
1207         'AndyAyersMS',
1208         'briansull',
1209         'BruceForstall',
1210         'CarolEidt',
1211         'cmckinsey',
1212         'erozenfeld',
1213         'janvorli',
1214         'jashook',
1215         'JosephTremoulet',
1216         'pgodeq',
1217         'pgavlin',
1218         'rartemev',
1219         'russellhadley',
1220         'RussKeldorph',
1221         'sandreenko',
1222         'sdmaclea',
1223         'swaroop-sridhar',
1224         'jkotas',
1225         'markwilkie',
1226         'weshaggard'
1227     ]
1228
1229     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1230     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1231     def osGroup = getOSGroup(os)
1232     switch (architecture) {
1233         case 'x64': // editor brace matching: {
1234             if (scenario == 'formatting') {
1235                 assert configuration == 'Checked'
1236                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1237                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1238                 }
1239
1240                 break
1241             }
1242
1243             switch (os) {
1244                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1245                 case 'Debian8.4':
1246                 case 'RHEL7.2':
1247                     if (scenario == 'innerloop') {
1248                         assert !isFlowJob
1249                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1250                     }
1251                     else if (scenario == 'normal') {
1252                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1253                     }
1254                     break
1255
1256                 case 'Ubuntu16.04':
1257                     assert !isFlowJob
1258                     assert scenario != 'innerloop'
1259                     // Distinguish with the other architectures (arm and x86)
1260                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1261                     break
1262
1263                 case 'Fedora24':
1264                 case 'Ubuntu16.10':
1265                     assert !isFlowJob
1266                     assert scenario != 'innerloop'
1267                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1268                     break
1269
1270                 case 'Ubuntu':
1271                     if (scenario == 'illink') {
1272                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1273                         break
1274                     }
1275                     // fall through
1276
1277                 case 'OSX10.12':
1278                     // Triggers on the non-flow jobs aren't necessary here
1279                     // Corefx testing uses non-flow jobs.
1280                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1281                         break
1282                     }
1283                     switch (scenario) {
1284                         case 'innerloop':
1285                             // PR Triggered jobs. These jobs will run pri0 tests.
1286                             if (configuration == 'Checked') {
1287                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1288                             }
1289                             break
1290
1291                         case 'normal':
1292                             // OSX uses checked for default PR tests
1293                             if (configuration == 'Checked') {
1294                                 // Default trigger
1295                                 assert !job.name.contains("centos")
1296                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1297                             }
1298                             break
1299
1300                         case 'jitdiff':
1301                             if (configuration == 'Checked') {
1302                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1303                             }
1304                             break
1305
1306                         case 'ilrt':
1307                             if (configuration == 'Release') {
1308                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1309                             }
1310                             break
1311
1312                         case 'longgc':
1313                             if (configuration == 'Release') {
1314                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1315                             }
1316                             break
1317
1318                         case 'gcsimulator':
1319                             if (configuration == 'Release') {
1320                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1321                             }
1322                             break
1323
1324                         case 'standalone_gc':
1325                             if (configuration == 'Release' || configuration == 'Checked') {
1326                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1327                             }
1328                             break
1329
1330                         case 'gc_reliability_framework':
1331                             if (configuration == 'Release' || configuration == 'Checked') {
1332                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1333                             }
1334                             break
1335
1336                         default:
1337                             if (isJitStressScenario(scenario)) {
1338                                 def displayStr = getStressModeDisplayName(scenario)
1339                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1340                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1341                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1342                             }
1343                             else if (isR2RScenario(scenario)) {
1344                                 if (configuration == 'Release' || configuration == 'Checked') {
1345                                     def displayStr = getR2RDisplayName(scenario)
1346                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1347                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1348                                 }
1349                             }
1350                             else {
1351                                 println("Unknown scenario: ${scenario}");
1352                                 assert false
1353                             }
1354                             break
1355
1356                     }
1357                     break
1358
1359                 case 'CentOS7.1':
1360                     switch (scenario) {
1361                         case 'innerloop':
1362                             // CentOS uses checked for default PR tests while debug is build only
1363                             if (configuration == 'Debug') {
1364                                 // Default trigger
1365                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1366                             }
1367
1368                             // Make sure this is a flow job to get build and test.
1369                             if (configuration == 'Checked' && isFlowJob) {
1370                                 assert job.name.contains("flow")
1371                                 // Default trigger
1372                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1373                             }
1374                             break
1375
1376                         case 'normal':
1377                             // Make sure this is a flow job to get build and test.
1378                             if (configuration == 'Checked' && isFlowJob) {
1379                                 assert job.name.contains("flow")
1380                                 // Default trigger
1381                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1382                             }
1383                             break
1384
1385                         default:
1386                             if (isR2RScenario(scenario)) {
1387                                 if (configuration == 'Release' || configuration == 'Checked') {
1388                                     def displayStr = getR2RDisplayName(scenario)
1389                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1390                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1391                                 }
1392                             }
1393                             break
1394
1395                     }
1396                     break
1397
1398                 case 'Windows_NT':
1399                     switch (scenario) {
1400                         case 'innerloop':
1401                             // Default trigger
1402                             if (configuration == 'Checked' || configuration == 'Release') {
1403                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1404                             }
1405                             break
1406
1407                         case 'normal':
1408                             if (configuration == 'Checked') {
1409                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1410                             }
1411                             break
1412
1413                         case 'jitdiff':
1414                             if (configuration == 'Checked') {
1415                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1416                             }
1417                             break
1418
1419                         case 'ilrt':
1420                             if (configuration == 'Release') {
1421                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1422                             }
1423                             break
1424
1425                         case 'longgc':
1426                             if (configuration == 'Release') {
1427                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1428                             }
1429                             break
1430
1431                         case 'gcsimulator':
1432                             if (configuration == 'Release') {
1433                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1434                             }
1435                             break
1436
1437                         case 'standalone_gc':
1438                             if (configuration == 'Release' || configuration == 'Checked') {
1439                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1440                             }
1441                             break
1442
1443                         case 'gc_reliability_framework':
1444                             if (configuration == 'Release' || configuration == 'Checked') {
1445                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1446                             }
1447                             break
1448
1449                         case 'illink':
1450                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1451                             break
1452
1453                         default:
1454                             if (isJitStressScenario(scenario)) {
1455                                 def displayStr = getStressModeDisplayName(scenario)
1456                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1457                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1458                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1459                             }
1460                             else if (isR2RScenario(scenario)) {
1461                                 if (configuration == 'Release' || configuration == 'Checked') {
1462                                     def displayStr = getR2RDisplayName(scenario)
1463                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1464                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1465                                 }
1466                             }
1467                             else {
1468                                 println("Unknown scenario: ${scenario}");
1469                                 assert false
1470                             }
1471                             break
1472
1473                     }
1474                     break
1475
1476                 default:
1477                     println("Unknown os: ${os}");
1478                     assert false
1479                     break
1480
1481             }
1482
1483             break
1484
1485         // editor brace matching: }
1486         case 'armlb':
1487         case 'arm': // editor brace matching: {
1488             switch (os) {
1489                 case 'Ubuntu':
1490                 case 'Ubuntu16.04':
1491                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1492                         break
1493                     }
1494
1495                     assert scenario != 'innerloop'
1496                     job.with {
1497                         publishers {
1498                             azureVMAgentPostBuildAction {
1499                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1500                             }
1501                         }
1502                     }
1503                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1504                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1505                     break
1506
1507                 case 'Tizen':
1508                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1509                         break
1510                     }
1511
1512                     architecture = 'armel'
1513                     job.with {
1514                         publishers {
1515                             azureVMAgentPostBuildAction {
1516                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1517                             }
1518                         }
1519                     }
1520
1521                     if (scenario == 'innerloop') {
1522                         if (configuration == 'Checked') {
1523                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1524                         }
1525                     }
1526                     else {
1527                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1528                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1529                     }
1530                     break
1531
1532                 case 'Windows_NT':
1533                     if (architecture == "armlb") {
1534                         // Disable armlb windows jobs
1535                         break
1536                     }
1537
1538                     // Triggers on the non-flow jobs aren't necessary here
1539                     if (!isFlowJob) {
1540                         break
1541                     }
1542
1543                     // Set up a private trigger
1544                     def contextString = "${os} ${architecture} Cross ${configuration}"
1545                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1546                     if (scenario == 'innerloop') {
1547                         contextString += " Innerloop"
1548                         triggerString += "\\W+Innerloop"
1549                     }
1550                     else {
1551                         contextString += " ${scenario}"
1552                         triggerString += "\\W+${scenario}"
1553                     }
1554
1555                     if (configuration == 'Debug') {
1556                         contextString += " Build"
1557                         triggerString += "\\W+Build"
1558                     } else {
1559                         contextString += " Build and Test"
1560                         triggerString += "\\W+Build and Test"
1561                     }
1562
1563                     triggerString += ".*"
1564
1565                     switch (scenario) {
1566                         case 'innerloop':
1567                             // Only Checked is an innerloop trigger.
1568                             if (configuration == 'Checked')
1569                             {
1570                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1571                             }
1572                             break
1573                         case 'normal':
1574                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1575                             break
1576                         default:
1577                             // Stress jobs will use this code path.
1578                             if (isArmWindowsScenario(scenario)) {
1579                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1580                             }
1581                             break
1582                     }
1583                     break
1584                 default:
1585                     println("NYI os: ${os}");
1586                     assert false
1587                     break
1588             }
1589             break
1590         // editor brace matching: }
1591         case 'arm64': // editor brace matching: {
1592             // Set up a private trigger
1593             def contextString = "${os} ${architecture} Cross ${configuration}"
1594             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1595
1596             if (scenario == 'innerloop') {
1597                 contextString += " Innerloop"
1598                 triggerString += "\\W+Innerloop"
1599             }
1600             else {
1601                 contextString += " ${scenario}"
1602                 triggerString += "\\W+${scenario}"
1603             }
1604
1605             if (configuration == 'Debug') {
1606                 contextString += " Build"
1607                 triggerString += "\\W+Build"
1608             } else {
1609                 contextString += " Build and Test"
1610                 triggerString += "\\W+Build and Test"
1611             }
1612
1613             triggerString += ".*"
1614
1615             switch (os) {
1616                 case 'Ubuntu':
1617                 case 'Ubuntu16.04':
1618                     switch (scenario) {
1619                         case 'innerloop':
1620                             if (configuration == 'Debug' && !isFlowJob) {
1621                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1622                             }
1623
1624                             break
1625                         case 'normal':
1626                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1627                             break
1628                         default:
1629                             if (isR2RScenario(scenario)) {
1630                                 if (configuration == 'Checked' || configuration == 'Release') {
1631                                     def displayStr = getR2RDisplayName(scenario)
1632                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1633                                 }
1634                             }
1635                             break
1636                     }
1637                     break
1638
1639                 case 'Windows_NT':
1640                     // Triggers on the non-flow jobs aren't necessary here
1641                     if (!isFlowJob) {
1642                         break
1643                     }
1644
1645                     assert isArmWindowsScenario(scenario)
1646                     switch (scenario) {
1647                         case 'innerloop':
1648                             if (configuration == 'Checked') {
1649                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1650                             }
1651
1652                             break
1653                         case 'normal':
1654                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1655                             break
1656                         default:
1657                             // Stress jobs will use this code path.
1658                             if (isArmWindowsScenario(scenario)) {
1659                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1660                             }
1661                             break
1662                     }
1663                     break
1664                 default:
1665                     println("NYI os: ${os}");
1666                     assert false
1667                     break
1668             }
1669             break
1670
1671         // editor brace matching: }
1672         case 'x86': // editor brace matching: {
1673             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1674             if (os == 'Ubuntu') {
1675                 // Triggers on the non-flow jobs aren't necessary here
1676                 if (!isFlowJob) {
1677                     break
1678                 }
1679
1680                 // on-demand only for ubuntu x86
1681                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1682                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1683                 break
1684
1685             }
1686             switch (scenario) {
1687                 case 'innerloop':
1688                     if (configuration == 'Checked' || configuration == 'Release') {
1689                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1690                     }
1691                     break
1692
1693                 case 'normal':
1694                     if (configuration == 'Checked') {
1695                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1696                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1697                     }
1698                     break
1699
1700                 case 'ilrt':
1701                     if (configuration == 'Release') {
1702                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1703                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1704                     }
1705                     break
1706
1707                 case 'longgc':
1708                     if (configuration == 'Release') {
1709                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1710                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1711                     }
1712                     break
1713
1714                 case 'gcsimulator':
1715                     if (configuration == 'Release') {
1716                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1717                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1718                     }
1719                     break
1720
1721                 case 'standalone_gc':
1722                     if (configuration == 'Release' || configuration == 'Checked') {
1723                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1724                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1725                     }
1726                     break
1727
1728                 case 'illink':
1729                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1730                     break
1731
1732                 default:
1733                     if (isJitStressScenario(scenario)) {
1734                         def displayStr = getStressModeDisplayName(scenario)
1735                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1736                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1737                     }
1738                     else if (isR2RScenario(scenario)) {
1739                         if (configuration == 'Release' || configuration == 'Checked') {
1740                             def displayStr = getR2RDisplayName(scenario)
1741                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1742                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1743                         }
1744                     }
1745                     else {
1746                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1747                         assert false
1748                     }
1749                     break
1750
1751             }
1752             break
1753
1754          // editor brace matching: }
1755         case 'x64_arm64_altjit':
1756         case 'x86_arm_altjit': // editor brace matching: {
1757             assert (os == 'Windows_NT')
1758             switch (scenario) {
1759                 case 'normal':
1760                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1761                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1762                     break
1763                 default:
1764                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1765                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1766                     break
1767             }
1768             break
1769
1770         // editor brace matching: }
1771         default:
1772             println("Unknown architecture: ${architecture}");
1773             assert false
1774             break
1775     }
1776 }
1777
1778 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1779     def buildCommands = [];
1780     def osGroup = getOSGroup(os)
1781     def lowerConfiguration = configuration.toLowerCase()
1782
1783     def priority = '1'
1784     if (scenario == 'innerloop') {
1785         priority = '0'
1786     }
1787
1788     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1789
1790     def enableCorefxTesting = isCoreFxScenario(scenario)
1791
1792     // Calculate the build steps, archival, and xunit results
1793     switch (os) {
1794         case 'Windows_NT': // editor brace matching: {
1795             switch (architecture) {
1796                 case 'x64':
1797                 case 'x86':
1798                 case 'x86_arm_altjit':
1799                 case 'x64_arm64_altjit':
1800                     def arch = architecture
1801                     def buildOpts = ''
1802                     if (architecture == 'x86_arm_altjit') {
1803                         arch = 'x86'
1804                     }
1805                     else if (architecture == 'x64_arm64_altjit') {
1806                         arch = 'x64'
1807                     }
1808
1809                     if (scenario == 'formatting') {
1810                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1811                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1812                         break
1813                     }
1814
1815                     if (scenario == 'illink') {
1816                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1817                     }
1818
1819                     // If it is a release build for windows, ensure PGO is used, else fail the build
1820                     if ((lowerConfiguration == 'release') &&
1821                         (scenario in Constants.basicScenarios) &&
1822                         (architecture != 'x86_arm_altjit') &&
1823                         (architecture != 'x64_arm64_altjit')) {
1824
1825                         buildOpts += ' -enforcepgo'
1826                     }
1827
1828                     if (enableCorefxTesting) {
1829                         buildOpts += ' skiptests';
1830                     } else {
1831                         buildOpts += " -priority=${priority}"
1832                     }
1833
1834                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1835                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1836                     // 35 characters long.
1837
1838                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1839
1840                     if (!isBuildOnly) {
1841                         def runtestArguments = ''
1842                         def testOpts = 'collectdumps'
1843
1844                         if (isR2RScenario(scenario)) {
1845
1846                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1847                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1848                             // to cause the tests to be crossgen'ed.
1849
1850                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1851                                 testOpts += ' crossgenaltjit protononjit.dll'
1852                             } else {
1853                                 testOpts += ' crossgen'
1854                             }
1855
1856                             testOpts += ' runcrossgentests'
1857                         }
1858                         else if (scenario == 'jitdiff') {
1859                             testOpts += ' jitdisasm crossgen'
1860                         }
1861                         else if (scenario == 'ilrt') {
1862                             testOpts += ' ilasmroundtrip'
1863                         }
1864                         else if (isLongGc(scenario)) {
1865                             testOpts += " ${scenario} sequential"
1866                         }
1867                         else if (scenario == 'standalone_gc') {
1868                             testOpts += ' gcname clrgc.dll'
1869                         }
1870                         else if (scenario == 'illink') {
1871                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1872                         }
1873
1874                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1875                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1876                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1877                         // which is read by the xunit harness.)
1878                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1879                         {
1880                             def timeout = 1800000
1881                             testOpts += " timeout ${timeout}"
1882                         }
1883
1884                         // If we are running a stress mode, we should write out the set of key
1885                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1886
1887                         def envScriptPath = ''
1888                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1889                             def buildCommandsStr = ''
1890                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1891                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1892
1893                             if (isJitStressScenario(scenario)) {
1894                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1895                             }
1896                             else if (isR2RStressScenario(scenario)) {
1897                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1898                             }
1899
1900                             if (architecture == 'x86_arm_altjit') {
1901                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1902                             }
1903                             else if (architecture == 'x64_arm64_altjit') {
1904                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1905                             }
1906
1907                             envScriptFinalize(os, envScriptPath)
1908
1909                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1910                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1911                             // into a single script.
1912                             buildCommands += buildCommandsStr
1913                         }
1914                         else if (architecture == 'x86_arm_altjit') {
1915                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1916                         }
1917                         else if (architecture == 'x64_arm64_altjit') {
1918                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1919                         }
1920                         if (envScriptPath != '') {
1921                             testOpts += " TestEnv ${envScriptPath}"
1922                         }
1923
1924                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1925
1926                         if (enableCorefxTesting) {
1927                             def workspaceRelativeFxRoot = "_/fx"
1928                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1929
1930                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1931
1932                             // Archive and process (only) the test results
1933                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1934                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1935
1936                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1937                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1938                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1939                         }
1940                         else if (isGcReliabilityFramework(scenario)) {
1941                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1942                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1943                         }
1944                         else {
1945                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1946                         }
1947                     }
1948
1949                     if (!enableCorefxTesting) {
1950                         // Run the rest of the build
1951                         // Build the mscorlib for the other OS's
1952                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1953                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1954
1955                         if (arch == "x64") {
1956                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1957                         }
1958
1959                         // Zip up the tests directory so that we don't use so much space/time copying
1960                         // 10s of thousands of files around.
1961                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1962
1963                         if (!isJitStressScenario(scenario)) {
1964                             // For Windows, pull full test results and test drops for x86/x64.
1965                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1966                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1967                         }
1968
1969                         if (scenario == 'jitdiff') {
1970                             // retrieve jit-dasm output for base commit, and run jit-diff
1971                             if (!isBuildOnly) {
1972                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1973                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1974                             }
1975                         }
1976
1977                         if (!isBuildOnly) {
1978                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1979                         }
1980                     }
1981                     break
1982                 case 'armlb':
1983                 case 'arm':
1984                     assert isArmWindowsScenario(scenario)
1985
1986                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1987                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1988
1989                     def buildArchitecture = 'arm'
1990
1991                     def buildOpts = ''
1992
1993                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1994                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1995                     if (architecture == 'armlb') {
1996                         buildOpts += ' -crossgenaltjit legacyjit.dll'
1997                     }
1998
1999                     if (enableCorefxTesting) {
2000                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2001                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2002                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2003                         //
2004                         // buildOpts += ' skiptests';
2005                         buildOpts += " -priority=0"
2006                     } else {
2007                         buildOpts += " -priority=${priority}"
2008                     }
2009
2010                     // This is now a build only job. Do not run tests. Use the flow job.
2011                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2012
2013                     if (enableCorefxTesting) {
2014                         assert isBuildOnly
2015                         assert architecture == 'arm'
2016
2017                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2018                         // to determine the correct matching corefx version git commit hash.
2019                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2020
2021                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2022                         def envScriptPath = ''
2023                         def buildCommandsStr = ''
2024                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2025                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2026                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2027                         envScriptFinalize(os, envScriptPath)
2028                         buildCommands += buildCommandsStr
2029
2030                         def workspaceRelativeFxRootLinux = "_/fx"
2031                         def workspaceRelativeFxRootWin = "_\\fx"
2032                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2033
2034                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2035
2036                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2037                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2038                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2039
2040                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2041                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2042                     } else {
2043                         // Zip up the tests directory so that we don't use so much space/time copying
2044                         // 10s of thousands of files around.
2045                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2046
2047                         // Add archival.
2048                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2049                     }
2050                     break
2051                 case 'arm64':
2052                     assert isArmWindowsScenario(scenario)
2053
2054                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2055                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2056
2057                     // This is now a build only job. Do not run tests. Use the flow job.
2058                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2059
2060                     // Zip up the tests directory so that we don't use so much space/time copying
2061                     // 10s of thousands of files around.
2062                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2063
2064                     // Add archival.
2065                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2066                     break
2067                 default:
2068                     println("Unknown architecture: ${architecture}");
2069                     assert false
2070                     break
2071             }
2072             break
2073         // editor brace matching: }
2074         case 'Ubuntu':
2075         case 'Ubuntu16.04':
2076         case 'Ubuntu16.10':
2077         case 'Debian8.4':
2078         case 'OSX10.12':
2079         case 'CentOS7.1':
2080         case 'RHEL7.2':
2081         case 'Tizen':
2082         case 'Fedora24': // editor brace matching: {
2083             switch (architecture) {
2084                 case 'x64':
2085                 case 'x86':
2086                     if (architecture == 'x86' && os == 'Ubuntu') {
2087                         // build and PAL test
2088                         def dockerImage = getDockerImageName(architecture, os, true)
2089                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2090                         dockerImage = getDockerImageName(architecture, os, false)
2091                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2092                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2093                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2094                         break
2095                     }
2096
2097                     if (scenario == 'formatting') {
2098                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2099                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2100                         break
2101                     }
2102
2103                     if (scenario == 'illink') {
2104                         assert(os == 'Ubuntu')
2105                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2106                     }
2107
2108                     if (!enableCorefxTesting) {
2109                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2110                         // only on supported OS platforms.
2111                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2112                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2113
2114                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2115                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2116
2117                         // Basic archiving of the build
2118                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2119                         // And pal tests
2120                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2121                     }
2122                     else {
2123                         // Corefx stress testing
2124                         assert os == 'Ubuntu'
2125                         assert architecture == 'x64'
2126                         assert lowerConfiguration == 'checked'
2127                         assert isJitStressScenario(scenario)
2128
2129                         // Build coreclr
2130                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2131
2132                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2133
2134                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2135                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2136                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2137                         buildCommands += envScriptCmds
2138
2139                         // Build and text corefx
2140                         def workspaceRelativeFxRoot = "_/fx"
2141                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2142
2143                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2144
2145                         // Archive and process (only) the test results
2146                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2147                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2148                     }
2149                     break
2150                 case 'arm64':
2151                     if (!enableCorefxTesting) {
2152                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2153
2154                         // HACK -- Arm64 does not have corefx jobs yet.
2155                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2156                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2157                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2158                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2159
2160                         // Basic archiving of the build
2161                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2162                     }
2163                     break
2164                 case 'arm':
2165                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2166                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2167
2168                     // default values for Ubuntu
2169                     def arm_abi="arm"
2170                     def linuxCodeName="trusty"
2171                     if (os == 'Ubuntu16.04') {
2172                         linuxCodeName="xenial"
2173                     }
2174                     else if (os == 'Tizen') {
2175                         arm_abi="armel"
2176                         linuxCodeName="tizen"
2177                     }
2178
2179                     // Unzip the Windows test binaries first. Exit with 0
2180                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2181
2182                     // Unpack the corefx binaries
2183                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2184                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2185                     if (os != 'Tizen') {
2186                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2187                     }
2188                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2189                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2190                     if (os == 'Ubuntu') {
2191                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2192                     }
2193
2194                     // Call the ARM CI script to cross build and test using docker
2195                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2196                     --mode=docker \\
2197                     --${arm_abi} \\
2198                     --linuxCodeName=${linuxCodeName} \\
2199                     --buildConfig=${lowerConfiguration} \\
2200                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2201                     --coreFxBinDir=./bin/CoreFxBinDir \\
2202                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2203
2204                     // Basic archiving of the build, no pal tests
2205                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2206                     break
2207                 default:
2208                     println("Unknown architecture: ${architecture}");
2209                     assert false
2210                     break
2211             }
2212             break
2213         // editor brace matching: }
2214         default:
2215             println("Unknown os: ${os}");
2216             assert false
2217             break
2218     } // os
2219
2220     return buildCommands
2221 }
2222
2223 Constants.allScenarios.each { scenario ->
2224     [true, false].each { isPR ->
2225         Constants.architectureList.each { architecture ->
2226             Constants.configurationList.each { configuration ->
2227                 Constants.osList.each { os ->
2228                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2229                     // and reset the os to Windows_NT
2230                     def isBuildOnly = false
2231                     if (os == 'Windows_NT_BuildOnly') {
2232                         isBuildOnly = true
2233                         os = 'Windows_NT'
2234                     }
2235
2236                     // Tizen is only supported for arm architecture
2237                     if (os == 'Tizen' && architecture != 'arm') {
2238                         return
2239                     }
2240
2241                     // Skip totally unimplemented (in CI) configurations.
2242                     switch (architecture) {
2243                         case 'arm64':
2244                             if (os == 'Ubuntu16.04') {
2245                                 os = 'Ubuntu'
2246                             }
2247
2248                             // Windows and Ubuntu only
2249                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2250                                 return
2251                             }
2252                             break
2253                         case 'arm':
2254                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2255                                 return
2256                             }
2257                             break
2258                         case 'armlb':
2259                             // Do not create armlb jobs
2260                             return
2261                             break
2262                         case 'x86':
2263                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2264                                 return
2265                             }
2266                             break
2267                         case 'x86_arm_altjit':
2268                         case 'x64_arm64_altjit':
2269                             if (os != 'Windows_NT') {
2270                                 return
2271                             }
2272                             break
2273                         case 'x64':
2274                             // Everything implemented
2275                             break
2276                         default:
2277                             println("Unknown architecture: ${architecture}")
2278                             assert false
2279                             break
2280                     }
2281
2282                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2283                     // with checked builds)
2284                     if (isJitStressScenario(scenario)) {
2285                         if (configuration != 'Checked') {
2286                             return
2287                         }
2288
2289                         // Since these are just execution time differences,
2290                         // skip platforms that don't execute the tests here (Windows_NT only)
2291                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2292                         if (!isEnabledOS) {
2293                             return
2294                         }
2295
2296                         switch (architecture) {
2297                             case 'x64':
2298                             case 'x86':
2299                             case 'x86_arm_altjit':
2300                             case 'x64_arm64_altjit':
2301                                 // x86 ubuntu: default only
2302                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2303                                     return
2304                                 }
2305                                 if (isBuildOnly) {
2306                                     return
2307                                 }
2308                                 break
2309
2310                             case 'arm':
2311                                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2312                                 if (!isBuildOnly || !isCoreFxScenario(scenario)) {
2313                                     return
2314                                 }
2315                                 break
2316
2317                             default:
2318                                 // arm64, armlb: stress is handled through flow jobs.
2319                                 return
2320                         }
2321                     }
2322                     else if (isR2RScenario(scenario)) {
2323                         if (os != 'Windows_NT') {
2324                             return
2325                         }
2326                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2327                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2328                             return
2329                         }
2330                     }
2331                     else {
2332                         // Skip scenarios
2333                         switch (scenario) {
2334                             case 'ilrt':
2335                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2336                                 // the default scenario build
2337                                 if (os != 'Windows_NT') {
2338                                     return
2339                                 }
2340                                 // Only x64 for now
2341                                 if (architecture != 'x64') {
2342                                     return
2343                                 }
2344                                 // Release only
2345                                 if (configuration != 'Release') {
2346                                     return
2347                                 }
2348                                 break
2349                             case 'jitdiff':
2350                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2351                                     return
2352                                 }
2353                                 if (architecture != 'x64') {
2354                                     return
2355                                 }
2356                                 if (configuration != 'Checked') {
2357                                     return
2358                                 }
2359                                 break
2360                             case 'longgc':
2361                             case 'gcsimulator':
2362                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2363                                     return
2364                                 }
2365                                 if (architecture != 'x64') {
2366                                     return
2367                                 }
2368                                 if (configuration != 'Release') {
2369                                     return
2370                                 }
2371                                 break
2372                             case 'gc_reliability_framework':
2373                             case 'standalone_gc':
2374                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2375                                     return
2376                                 }
2377
2378                                 if (architecture != 'x64') {
2379                                     return
2380                                 }
2381
2382                                 if (configuration != 'Release' && configuration != 'Checked') {
2383                                     return
2384                                 }
2385                                 break
2386                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2387                             case 'formatting':
2388                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2389                                     return
2390                                 }
2391                                 if (architecture != 'x64') {
2392                                     return
2393                                 }
2394                                 if (configuration != 'Checked') {
2395                                     return
2396                                 }
2397                                 if (isBuildOnly) {
2398                                     return
2399                                 }
2400                                 break
2401                             case 'illink':
2402                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2403                                     return
2404                                 }
2405                                 if (architecture != 'x64' && architecture != 'x86') {
2406                                     return
2407                                 }
2408                                 if (isBuildOnly) {
2409                                     return
2410                                 }
2411                                 break
2412                             case 'normal':
2413                                 // Nothing skipped
2414                                 break
2415                             case 'innerloop':
2416                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2417                                     return
2418                                 }
2419                                 break
2420                             default:
2421                                 println("Unknown scenario: ${scenario}")
2422                                 assert false
2423                                 break
2424                         }
2425                     }
2426
2427                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2428                     // runtime behavior, not compile-time behavior, are not interesting.
2429                     switch (architecture) {
2430                         case 'x86_arm_altjit':
2431                         case 'x64_arm64_altjit':
2432                             if (isGCStressRelatedTesting(scenario)) {
2433                                 return
2434                             }
2435                             break
2436                         default:
2437                             break
2438                     }
2439
2440                     // Calculate names
2441                     def lowerConfiguration = configuration.toLowerCase()
2442                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2443                     def folderName = getJobFolder(scenario)
2444
2445                     // Create the new job
2446                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2447                     addToViews(newJob, isPR, architecture, os)
2448
2449                     def machineAffinityOptions = null
2450
2451                     if (os != 'Windows_NT') {
2452                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2453                     }
2454                     else {
2455                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2456                     }
2457
2458                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2459
2460                     // Add all the standard options
2461                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2462                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2463
2464                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2465                     def osGroup = getOSGroup(os)
2466
2467                     newJob.with {
2468                         steps {
2469                             if (os == 'Windows_NT') {
2470                                 buildCommands.each { buildCommand ->
2471                                     batchFile(buildCommand)
2472                                 }
2473                             }
2474                             else {
2475                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2476                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2477                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2478                                     // Define the Windows Tests and Corefx build job names
2479                                     def WindowsTestsName = projectFolder + '/' +
2480                                                            Utilities.getFullJobName(project,
2481                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2482                                                                                     false)
2483                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2484                                                        Utilities.getFolderName(branch)
2485
2486                                     // Copy the Windows test binaries and the Corefx build binaries
2487                                     copyArtifacts(WindowsTestsName) {
2488                                         includePatterns('bin/tests/tests.zip')
2489                                         buildSelector {
2490                                             latestSuccessful(true)
2491                                         }
2492                                     }
2493
2494                                     def arm_abi = 'arm'
2495                                     def corefx_os = 'linux'
2496                                     if (os == 'Tizen') {
2497                                         arm_abi = 'armel'
2498                                         corefx_os = 'tizen'
2499                                     }
2500
2501                                     // Let's use release CoreFX to test checked CoreCLR,
2502                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2503                                     def corefx_lowerConfiguration = lowerConfiguration
2504                                     if ( lowerConfiguration == 'checked' ) {
2505                                         corefx_lowerConfiguration='release'
2506                                     }
2507
2508                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2509                                         includePatterns('bin/build.tar.gz')
2510                                         buildSelector {
2511                                             latestSuccessful(true)
2512                                         }
2513                                     }
2514                                 }
2515
2516                                 buildCommands.each { buildCommand ->
2517                                     shell(buildCommand)
2518                                 }
2519                             }
2520                         }
2521                     } // newJob.with
2522
2523                 } // os
2524             } // configuration
2525         } // architecture
2526     } // isPR
2527 } // scenario
2528
2529
2530 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2531 Constants.allScenarios.each { scenario ->
2532     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2533
2534     [true, false].each { isPR ->
2535         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2536             Constants.crossList.each { os ->
2537                 if (architecture == 'arm64') {
2538                     if (os != "Ubuntu" && os != "Windows_NT") {
2539                         return
2540                     }
2541                 } else if (architecture == 'arm') {
2542                     if (os != 'Windows_NT') {
2543                         return
2544                     }
2545                 } else if (architecture == 'armlb') {
2546                   // Do not create armlb windows jobs.
2547                   return
2548                 } else if (architecture == 'x86') {
2549                     if (os != "Ubuntu") {
2550                         return
2551                     }
2552                 }
2553
2554                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2555
2556                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2557                     return
2558                 }
2559
2560                 Constants.configurationList.each { configuration ->
2561
2562                     // First, filter based on OS.
2563
2564                     if (os == 'Windows_NT') {
2565                         if (!isArmWindowsScenario(scenario)) {
2566                             return
2567                         }
2568                     }
2569                     else {
2570                         // Non-Windows
2571                         if (architecture == 'arm64') {
2572                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2573                                 return
2574                             }
2575                         }
2576                         else if (architecture == 'x86') {
2577                             // Linux/x86 only want innerloop and default test
2578                             if (!isNormalOrInnerloop) {
2579                                 return
2580                             }
2581                         }
2582                     }
2583
2584                     // For CentOS, we only want Checked/Release builds.
2585                     if (os == 'CentOS7.1') {
2586                         if (configuration != 'Checked' && configuration != 'Release') {
2587                             return
2588                         }
2589                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2590                             return
2591                         }
2592                     }
2593
2594                     // For RedHat and Debian, we only do Release builds.
2595                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2596                         if (configuration != 'Release') {
2597                             return
2598                         }
2599                         if (!isNormalOrInnerloop) {
2600                             return
2601                         }
2602                     }
2603
2604                     // Next, filter based on scenario.
2605
2606                     if (isJitStressScenario(scenario)) {
2607                         if (configuration != 'Checked') {
2608                             return
2609                         }
2610
2611                         // CoreFx JIT stress tests currently only implemented for ARM.
2612                         if (isCoreFxScenario(scenario) && (architecture != 'arm')) {
2613                             return
2614                         }
2615                     }
2616                     else if (isR2RBaselineScenario(scenario)) {
2617                         if (configuration != 'Checked' && configuration != 'Release') {
2618                             return
2619                         }
2620                     }
2621                     else if (isR2RStressScenario(scenario)) {
2622                         if (configuration != 'Checked') {
2623                             return
2624                         }
2625                     }
2626                     else {
2627                         // Skip scenarios
2628                         switch (scenario) {
2629                             case 'ilrt':
2630                             case 'longgc':
2631                             case 'gcsimulator':
2632                                 // Long GC tests take a long time on non-Release builds
2633                                 // ilrt is also Release only
2634                                 if (configuration != 'Release') {
2635                                     return
2636                                 }
2637                                 break
2638
2639                             case 'jitdiff':
2640                                 if (configuration != 'Checked') {
2641                                     return;
2642                                 }
2643                                 break
2644
2645                             case 'gc_reliability_framework':
2646                             case 'standalone_gc':
2647                                 if (configuration != 'Release' && configuration != 'Checked') {
2648                                     return
2649                                 }
2650                                 break
2651
2652                             case 'formatting':
2653                                 return
2654                             case 'illink':
2655                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2656                                     return
2657                                 }
2658                                 break
2659
2660                             case 'normal':
2661                                 // Nothing skipped
2662                                 break
2663
2664                             case 'innerloop':
2665                                 // Nothing skipped
2666                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2667                                     return
2668                                 }
2669                                 break
2670
2671                             default:
2672                                 println("Unknown scenario: ${scenario}")
2673                                 assert false
2674                                 break
2675                         }
2676                     }
2677
2678                     // Done filtering. Now, create the jobs.
2679
2680                     // =============================================================================================
2681                     // Create the test job
2682                     // =============================================================================================
2683
2684                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2685
2686                     def lowerConfiguration = configuration.toLowerCase()
2687                     def osGroup = getOSGroup(os)
2688                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2689
2690                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2691                     def inputCoreCLRBuildIsBuildOnly = false
2692                     if (isCoreFxScenario(scenario)) {
2693                         // Every CoreFx test depends on its own unique build.
2694                         inputCoreCLRBuildScenario = scenario
2695                         inputCoreCLRBuildIsBuildOnly = true
2696                     }
2697                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
2698                     def inputCoreCLRBuildName = projectFolder + '/' +
2699                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
2700
2701                     def inputWindowsTestsBuildName = ""
2702                     if (windowsArmJob != true) {
2703                         // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2704                         // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2705
2706                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2707
2708                         def inputWindowsTestBuildArch = architecture
2709                         if (architecture == "arm64" && os != "Windows_NT") {
2710                             // Use the x64 test build for arm64 unix
2711                             inputWindowsTestBuildArch = "x64"
2712                         }
2713
2714                         if (isJitStressScenario(scenario)) {
2715                             inputWindowsTestsBuildName = projectFolder + '/' +
2716                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2717                         } else {
2718                             inputWindowsTestsBuildName = projectFolder + '/' +
2719                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2720                         }
2721                     } // if (windowsArmJob != true)
2722
2723                     def serverGCString = ''
2724                     def testOpts = ''
2725
2726                     if (windowsArmJob != true) {
2727                         // Enable Server GC for Ubuntu PR builds
2728                         if (os == 'Ubuntu' && isPR) {
2729                             serverGCString = '--useServerGC'
2730                         }
2731
2732                         if (isR2RScenario(scenario)) {
2733
2734                             testOpts += ' --crossgen --runcrossgentests'
2735
2736                             if (scenario == 'r2r_jitstress1') {
2737                                 testOpts += ' --jitstress=1'
2738                             }
2739                             else if (scenario == 'r2r_jitstress2') {
2740                                 testOpts += ' --jitstress=2'
2741                             }
2742                             else if (scenario == 'r2r_jitstressregs1') {
2743                                 testOpts += ' --jitstressregs=1'
2744                             }
2745                             else if (scenario == 'r2r_jitstressregs2') {
2746                                 testOpts += ' --jitstressregs=2'
2747                             }
2748                             else if (scenario == 'r2r_jitstressregs3') {
2749                                 testOpts += ' --jitstressregs=3'
2750                             }
2751                             else if (scenario == 'r2r_jitstressregs4') {
2752                                 testOpts += ' --jitstressregs=4'
2753                             }
2754                             else if (scenario == 'r2r_jitstressregs8') {
2755                                 testOpts += ' --jitstressregs=8'
2756                             }
2757                             else if (scenario == 'r2r_jitstressregs0x10') {
2758                                 testOpts += ' --jitstressregs=0x10'
2759                             }
2760                             else if (scenario == 'r2r_jitstressregs0x80') {
2761                                 testOpts += ' --jitstressregs=0x80'
2762                             }
2763                             else if (scenario == 'r2r_jitstressregs0x1000') {
2764                                 testOpts += ' --jitstressregs=0x1000'
2765                             }
2766                             else if (scenario == 'r2r_jitminopts') {
2767                                 testOpts += ' --jitminopts'
2768                             }
2769                             else if (scenario == 'r2r_jitforcerelocs') {
2770                                 testOpts += ' --jitforcerelocs'
2771                             }
2772                             else if (scenario == 'r2r_gcstress15') {
2773                                 testOpts += ' --gcstresslevel=0xF'
2774                             }
2775                         }
2776                         else if (scenario == 'jitdiff') {
2777                             testOpts += ' --jitdisasm --crossgen'
2778                         }
2779                         else if (scenario == 'illink') {
2780                             testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2781                         }
2782                         else if (isLongGc(scenario)) {
2783                             // Long GC tests behave very poorly when they are not
2784                             // the only test running (many of them allocate until OOM).
2785                             testOpts += ' --sequential'
2786
2787                             // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2788                             // for running long GC and GCSimulator tests, respectively. We don't use them
2789                             // here because using a playlist file produces much more readable output on the CI machines
2790                             // and reduces running time.
2791                             //
2792                             // The Long GC playlist contains all of the tests that are
2793                             // going to be run. The GCSimulator playlist contains all of
2794                             // the GC simulator tests.
2795                             if (scenario == 'longgc') {
2796                                 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2797                             }
2798                             else if (scenario == 'gcsimulator') {
2799                                 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2800                             }
2801                         }
2802                         else if (isGcReliabilityFramework(scenario)) {
2803                             testOpts += ' --build-overlay-only'
2804                         }
2805                         else if (scenario == 'standalone_gc') {
2806                             if (osGroup == 'OSX') {
2807                                 testOpts += ' --gcname=libclrgc.dylib'
2808                             }
2809                             else if (osGroup == 'Linux') {
2810                                 testOpts += ' --gcname=libclrgc.so'
2811                             }
2812                             else {
2813                                 println("Unexpected OS group: ${osGroup} for os ${os}")
2814                                 assert false
2815                             }
2816                         }
2817                     } // if (windowsArmJob != true)
2818
2819                     def folder = getJobFolder(scenario)
2820                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2821                         // Add parameters for the inputs
2822
2823                         if (windowsArmJob == true) {
2824                             parameters {
2825                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2826                             }
2827                         }
2828                         else {
2829                             parameters {
2830                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
2831                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2832                             }
2833                         }
2834
2835                         steps {
2836                             // Set up the copies
2837
2838                             // Coreclr build containing the tests and mscorlib
2839                             // pri1 jobs still need to copy windows_nt built tests
2840                             if (windowsArmJob != true) {
2841                                 copyArtifacts(inputWindowsTestsBuildName) {
2842                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2843                                     buildSelector {
2844                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2845                                     }
2846                                 }
2847                             }
2848
2849                             // Coreclr build we are trying to test
2850                             //
2851                             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2852
2853                             copyArtifacts(inputCoreCLRBuildName) {
2854                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2855                                 buildSelector {
2856                                     buildNumber('${CORECLR_BUILD}')
2857                                 }
2858                             }
2859
2860                             // Windows CoreCLR Arm(64) will restore corefx
2861                             // packages correctly.
2862                             //
2863                             // In addition, test steps are entirely different
2864                             // because we do not have a unified runner
2865                             if (windowsArmJob != true) {
2866                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2867
2868                                 // HACK -- Arm64 does not have corefx jobs yet.
2869                                 // Clone corefx and build the native packages overwriting the x64 packages.
2870                                 if (architecture == 'arm64') {
2871                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2872                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2873                                 }
2874                                 else if (architecture == 'x86') {
2875                                     shell("mkdir ./bin/CoreFxNative")
2876
2877                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2878                                         includePatterns('bin/build.tar.gz')
2879                                         targetDirectory('bin/CoreFxNative')
2880                                         buildSelector {
2881                                             latestSuccessful(true)
2882                                         }
2883                                     }
2884
2885                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2886                                 }
2887
2888                                 // Unzip the tests first.  Exit with 0
2889                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2890                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2891
2892                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2893
2894                                 // Execute the tests
2895                                 def runDocker = isNeedDocker(architecture, os, false)
2896                                 def dockerPrefix = ""
2897                                 def dockerCmd = ""
2898                                 if (runDocker) {
2899                                     def dockerImage = getDockerImageName(architecture, os, false)
2900                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2901                                     dockerCmd = dockerPrefix + "${dockerImage} "
2902                                 }
2903
2904                                 // If we are running a stress mode, we'll set those variables first
2905                                 def testEnvOpt = ""
2906                                 if (isJitStressScenario(scenario)) {
2907                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2908                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2909                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2910                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2911                                     shell("${envScriptCmds}")
2912                                     testEnvOpt = "--test-env=" + scriptFileName
2913                                 }
2914
2915                                 if (isGCStressRelatedTesting(scenario)) {
2916                                     shell('./init-tools.sh')
2917                                 }
2918
2919                                 shell("""${dockerCmd}./tests/runtest.sh \\
2920                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2921                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2922                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2923                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2924
2925                                 if (isGcReliabilityFramework(scenario)) {
2926                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2927                                     if (serverGCString != '') {
2928                                         if (runDocker) {
2929                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2930                                         }
2931                                         else {
2932                                             shell("export COMPlus_gcServer=1")
2933                                         }
2934                                     }
2935
2936                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2937                                 }
2938                             }
2939                             else { // windowsArmJob == true
2940
2941                                 if (isCoreFxScenario(scenario)) {
2942
2943                                     // Only arm supported for corefx testing now.
2944                                     assert architecture == 'arm'
2945
2946                                     // Unzip CoreFx runtime
2947                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2948
2949                                     // Unzip CoreFx tests.
2950                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2951
2952                                     // Add the script to run the corefx tests
2953                                     def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2954                                     def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2955                                     def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2956                                     batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2957
2958                                 } else { // !isCoreFxScenario(scenario)
2959
2960                                     // Unzip tests.
2961                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2962
2963                                     def buildCommands = ""
2964
2965                                     def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2966                                     def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2967                                     def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2968
2969                                     // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2970                                     addCommand("SETLOCAL ENABLEEXTENSIONS")
2971
2972                                     // For all jobs
2973                                     addEnvVariable("CORE_ROOT", coreRootLocation)
2974
2975                                     addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2976                                     addEnvVariable("COMPlus_ContinueOnAssert", "0")
2977
2978                                     // ARM legacy backend; this is an altjit.
2979                                     if (architecture == "armlb") {
2980                                         addEnvVariable("COMPlus_AltJit", "*")
2981                                         addEnvVariable("COMPlus_AltJitNgen", "*")
2982                                         addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2983                                         addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2984                                     }
2985
2986                                     // If we are running a stress mode, we'll set those variables as well
2987                                     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2988                                         def stressValues = null
2989                                         if (isJitStressScenario(scenario)) {
2990                                             stressValues = Constants.jitStressModeScenarios[scenario]
2991                                         }
2992                                         else {
2993                                             stressValues = Constants.r2rStressScenarios[scenario]
2994                                         }
2995
2996                                         stressValues.each { key, value ->
2997                                             addEnvVariable(key, value)
2998                                         }
2999                                     }
3000
3001                                     if (isR2RScenario(scenario)) {
3002                                         // Crossgen the framework assemblies.
3003                                         buildCommands += """
3004 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3005 @goto skip_PrecompileAssembly
3006
3007 :PrecompileAssembly
3008 @REM Skip mscorlib since it is already precompiled.
3009 @if /I "%3" == "mscorlib.dll" exit /b 0
3010 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3011
3012 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3013 @if "%errorlevel%" == "-2146230517" (
3014     echo %2 is not a managed assembly.
3015 ) else if "%errorlevel%" == "-2146234344" (
3016     echo %2 is not a managed assembly.
3017 ) else if %errorlevel% neq 0 (
3018     echo Unable to precompile %2
3019 ) else (
3020     echo Precompiled %2
3021 )
3022 @exit /b 0
3023
3024 :skip_PrecompileAssembly
3025 """
3026
3027                                         // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3028                                         // crossgen on tests before running them.
3029                                         addEnvVariable("RunCrossGen", "true")
3030                                     } // isR2RScenario(scenario)
3031
3032                                     // Create the smarty command
3033                                     def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3034                                     def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3035                                     def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3036
3037                                     def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
3038
3039                                     if (architecture == "armlb") {
3040                                         addExclude("LEGACYJIT_FAIL")
3041                                     }
3042
3043                                     // Exclude tests based on scenario.
3044                                     Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3045                                         addArchSpecificExclude(architecture, excludeTag)
3046                                     }
3047
3048                                     // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
3049                                     if (scenario == 'innerloop') {
3050                                         addExclude("pri1")
3051                                     }
3052
3053                                     // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3054                                     // TODO: We should create a "long running" job that runs these with a longer timeout.
3055                                     addExclude("LONG_RUNNING")
3056
3057                                     smartyCommand += "/lstFile Tests.lst"
3058
3059                                     def testListArch = [
3060                                         'arm64': 'arm64',
3061                                         'arm': 'arm',
3062                                         'armlb': 'arm'
3063                                     ]
3064
3065                                     def archLocation = testListArch[architecture]
3066
3067                                     addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3068                                     addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3069                                     addCommand("${smartyCommand}")
3070
3071                                     // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3072                                     // However, we also need to remove all the variables that were set during this batch file, so we
3073                                     // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3074                                     // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3075                                     // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3076                                     // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3077                                     // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3078                                     addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3079
3080                                     // ZIP up the smarty output, no matter what the smarty result.
3081                                     addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3082
3083                                     addCommand("echo %errorlevel%")
3084                                     addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3085
3086                                     // Use the smarty errorlevel as the script errorlevel.
3087                                     addCommand("exit /b %__save_smarty_errorlevel%")
3088
3089                                     batchFile(buildCommands)
3090                                 } // non-corefx testing
3091                             } // windowsArmJob == true
3092                         } // steps
3093                     } // job
3094
3095                     addToViews(newJob, isPR, architecture, os)
3096
3097                     if (scenario == 'jitdiff') {
3098                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3099                     }
3100
3101                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3102                     if (os in ['Ubuntu']) {
3103                         SummaryBuilder summaries = new SummaryBuilder()
3104                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3105                         summaries.emit(newJob)
3106                     }
3107
3108                     def affinityOptions = null
3109
3110                     if (windowsArmJob == true) {
3111                         affinityOptions = [
3112                             "use_arm64_build_machine" : false
3113                         ]
3114                     }
3115
3116                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3117                         affinityOptions = [
3118                             "large_pages" : false
3119                         ]
3120                     }
3121
3122                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3123                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3124
3125                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3126
3127                     if (windowsArmJob != true) {
3128                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3129                     }
3130                     else {
3131                         if (!isCoreFxScenario(scenario)) {
3132                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3133
3134                             // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3135                             // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3136                             // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3137                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3138                         }
3139                     }
3140
3141                     // =============================================================================================
3142                     // Create a build flow to join together the build and tests required to run this test.
3143                     // =============================================================================================
3144
3145                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3146                     // Linux CoreCLR test
3147                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3148                     def fullTestJobName = projectFolder + '/' + newJob.name
3149                     // Add a reference to the input jobs for report purposes
3150                     JobReport.Report.addReference(inputCoreCLRBuildName)
3151                     if (windowsArmJob != true) {
3152                         JobReport.Report.addReference(inputWindowsTestsBuildName)
3153                     }
3154                     JobReport.Report.addReference(fullTestJobName)
3155                     def newFlowJob = null
3156
3157                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3158                         // Do not create the flow job for RHEL jobs.
3159                         return
3160                     }
3161
3162                     // For pri0 jobs we can build tests on unix
3163                     if (windowsArmJob) {
3164                         // For Windows arm jobs there is no reason to build a parallel test job.
3165                         // The product build supports building and archiving the tests.
3166
3167                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3168                         buildFlow("""\
3169 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3170
3171 // And then build the test build
3172 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3173 """)
3174                         }
3175                     }
3176                     else {
3177                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3178                         buildFlow("""\
3179 // Build the input jobs in parallel
3180 parallel (
3181 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3182 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3183 )
3184
3185 // And then build the test build
3186 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3187             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3188 """)
3189                         }
3190                     }
3191
3192                     addToViews(newFlowJob, isPR, architecture, os)
3193
3194                     // For the flow jobs set the machine affinity as x64 if an armarch.
3195                     def flowArch = architecture
3196
3197                     if (flowArch in validWindowsNTCrossArches) {
3198                         flowArch = 'x64'
3199                         affinityOptions = null
3200                     }
3201
3202                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3203                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3204                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3205                 } // configuration
3206             } // os
3207         } // architecture
3208     } // isPR
3209 } // scenario
3210
3211 JobReport.Report.generateJobReport(out)
3212
3213 // Make the call to generate the help job
3214 Utilities.createHelperJob(this, project, branch,
3215     "Welcome to the ${project} Repository",  // This is prepended to the help message
3216     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3217
3218 Utilities.addCROSSCheck(this, project, branch)