netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'jitincompletehwintrinsic'
 266                // 'jitx86hwintrinsicnoavx'
 267                // 'jitx86hwintrinsicnoavx2'
 268                // 'jitx86hwintrinsicnosimd'
 269                // 'jitnox86hwintrinsic'
 270                'corefx_baseline':                        [],    // corefx tests don't use smarty
 271                'corefx_minopts':                         [],    // corefx tests don't use smarty
 272                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 273                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 274                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 275                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 276                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 277                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 278                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 279                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 280                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 281                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 283                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 284                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 285                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 286                'heapverify1':                            [],
 287                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 288                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 290                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 293
 294                //
 295                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 296                //
 297
 298                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 299                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 317     ]
 318
 319     def static validLinuxArm64Scenarios = [
 320         'normal',
 321         'r2r',
 322         'innerloop',
 323         'gcstress0x3',
 324         'gcstress0xc'
 325     ]
 326
 327     def static configurationList = ['Debug', 'Checked', 'Release']
 328
 329     // This is the set of architectures
 330     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 331 }
 332
 333 // **************************************************************
 334 // Create some specific views
 335 //
 336 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 337 // views based on a single regular expression. These views will be generated by adding a
 338 // specific set of jobs to them.
 339 //
 340 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 341 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 342 // those are required, don't add them (which simplifies the view pages, as well).
 343 // **************************************************************
 344
 345 class Views {
 346     def static MergeJobView = null
 347     def static PeriodicJobView = null
 348     def static ArchitectureViews = [:]
 349     def static OSViews = [:]
 350 }
 351
 352 // MergeJobView: include all jobs that execute when a PR change is merged.
 353 Views.MergeJobView = listView('Merge') {
 354     recurse()
 355     columns {
 356         status()
 357         weather()
 358         name()
 359         lastSuccess()
 360         lastFailure()
 361         lastDuration()
 362         buildButton()
 363     }
 364 }
 365
 366 // PeriodicJobView: include all jobs that execute on a schedule
 367 Views.PeriodicJobView = listView('Periodic') {
 368     recurse()
 369     columns {
 370         status()
 371         weather()
 372         name()
 373         lastSuccess()
 374         lastFailure()
 375         lastDuration()
 376         buildButton()
 377     }
 378 }
 379
 380 // Create a view for non-PR jobs for each architecture.
 381 Constants.architectureList.each { architecture ->
 382     Views.ArchitectureViews[architecture] = listView(architecture) {
 383         recurse()
 384         columns {
 385             status()
 386             weather()
 387             name()
 388             lastSuccess()
 389             lastFailure()
 390             lastDuration()
 391             buildButton()
 392         }
 393     }
 394 }
 395
 396 // Create a view for non-PR jobs for each OS.
 397 Constants.osList.each { os ->
 398     // Don't create one for the special 'Windows_NT_BuildOnly'
 399     if (os == 'Windows_NT_BuildOnly') {
 400         return
 401     }
 402     Views.OSViews[os] = listView(os) {
 403         recurse()
 404         columns {
 405             status()
 406             weather()
 407             name()
 408             lastSuccess()
 409             lastFailure()
 410             lastDuration()
 411             buildButton()
 412         }
 413     }
 414 }
 415
 416 def static addToMergeView(def job) {
 417     Views.MergeJobView.with {
 418         jobs {
 419             name(job.name)
 420         }
 421     }
 422 }
 423
 424 def static addToPeriodicView(def job) {
 425     Views.PeriodicJobView.with {
 426         jobs {
 427             name(job.name)
 428         }
 429     }
 430 }
 431
 432 def static addToViews(def job, def isPR, def architecture, def os) {
 433     if (isPR) {
 434         // No views want PR jobs currently.
 435         return
 436     }
 437
 438     // Add to architecture view.
 439     Views.ArchitectureViews[architecture].with {
 440         jobs {
 441             name(job.name)
 442         }
 443     }
 444
 445     // Add to OS view.
 446     Views.OSViews[os].with {
 447         jobs {
 448             name(job.name)
 449         }
 450     }
 451 }
 452
 453 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 454     addToPeriodicView(job)
 455     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 456 }
 457
 458 def static addGithubPushTriggerHelper(def job) {
 459     addToMergeView(job)
 460     Utilities.addGithubPushTrigger(job)
 461 }
 462
 463
 464 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 465     assert os instanceof String
 466     assert architecture instanceof String
 467
 468     def armArches = ['arm', 'armlb', 'arm64']
 469     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 470
 471     if (!(architecture in armArches)) {
 472         assert options == null
 473         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 474
 475         return
 476     }
 477
 478     // This is an arm(64) job.
 479     //
 480     // There are several options.
 481     //
 482     // Windows_NT
 483     //
 484     // Arm32 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 486     // Arm32 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 488     //
 489     // Arm64 (Build) -> latest-arm64
 490     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 491     // Arm64 (Test)  -> arm64-windows_nt
 492     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 493     //
 494     // Ubuntu
 495     //
 496     // Arm32 (Build) -> arm-cross-latest
 497     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 498     // Arm32 (Test)  -> NYI Arch not supported
 499     //       |->
 500     //
 501     // Arm64 (Build) -> arm64-cross-latest
 502     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 503     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 504     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 505     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 506     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 507
 508     // This has to be a arm arch
 509     assert architecture in armArches
 510     if (os == "Windows_NT") {
 511         // Arm(64) Windows jobs share the same machines for now
 512         def isBuild = options['use_arm64_build_machine'] == true
 513
 514         if (isBuild == true) {
 515             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 516         } else {
 517             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 518         }
 519     } else {
 520         assert os != 'Windows_NT'
 521         assert os in supportedArmLinuxOs
 522
 523         if (architecture == 'arm' || architecture == 'armlb') {
 524             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 525         } else {
 526             // Arm64 Linux
 527             if (options['is_build_only'] == true) {
 528                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 529             } else {
 530                 // Arm64 Test Machines
 531                 if (options['large_pages'] == false) {
 532                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 533                 } else {
 534                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 535                 }
 536             }
 537         }
 538     }
 539 }
 540
 541 def static isGCStressRelatedTesting(def scenario) {
 542     // The 'r2r_gcstress15' scenario is a basic scenario.
 543     // Detect it and make it a GCStress related.
 544     if (scenario == 'r2r_gcstress15')
 545     {
 546         return true;
 547     }
 548
 549     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 550     def scenarioName = scenario.toLowerCase()
 551     def isGCStressTesting = false
 552     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 553         if (k in gcStressTestEnvVars) {
 554             isGCStressTesting = true;
 555         }
 556     }
 557     return isGCStressTesting
 558 }
 559
 560 def static isCoreFxScenario(def scenario) {
 561     def corefx_prefix = 'corefx_'
 562     if (scenario.length() < corefx_prefix.length()) {
 563         return false
 564     }
 565     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 566 }
 567
 568 def static isR2RBaselineScenario(def scenario) {
 569     return (scenario == 'r2r')
 570 }
 571
 572 def static isR2RStressScenario(def scenario) {
 573     return Constants.r2rStressScenarios.containsKey(scenario)
 574 }
 575
 576 def static isR2RScenario(def scenario) {
 577     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 578 }
 579
 580 def static isJitStressScenario(def scenario) {
 581     return Constants.jitStressModeScenarios.containsKey(scenario)
 582 }
 583
 584 def static isLongGc(def scenario) {
 585     return (scenario == 'longgc' || scenario == 'gcsimulator')
 586 }
 587
 588 def static isJitDiff(def scenario) {
 589     return (scenario == 'jitdiff')
 590 }
 591
 592 def static isGcReliabilityFramework(def scenario) {
 593     return (scenario == 'gc_reliability_framework')
 594 }
 595
 596 def static isArmWindowsScenario(def scenario) {
 597     return Constants.validArmWindowsScenarios.containsKey(scenario)
 598 }
 599
 600 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 601     if (isBuildOnly == true) {
 602         os = 'Windows_NT_buildOnly'
 603     }
 604
 605     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 606
 607     if (validOsPrTriggerArchConfigs == null) {
 608         return false
 609     }
 610
 611     if (validOsPrTriggerArchConfigs[architecture] != null) {
 612         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 613
 614         if (!(configuration in validOsPrTriggerConfigs)) {
 615             return false
 616         }
 617     } else {
 618         return false
 619     }
 620
 621     return true
 622 }
 623
 624 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 625     // 2 hours (120 minutes) is the default timeout
 626     def timeout = 120
 627     def innerLoop = (scenario == "innerloop")
 628
 629     if (!innerLoop) {
 630         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 631         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 632         timeout = 240
 633     }
 634
 635     if (!isBuildOnly) {
 636         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 637         if (isGCStressRelatedTesting(scenario)) {
 638             timeout = 4320
 639         }
 640         else if (isCoreFxScenario(scenario)) {
 641             timeout = 360
 642         }
 643         else if (isJitStressScenario(scenario)) {
 644             timeout = 300
 645         }
 646         else if (isR2RBaselineScenario(scenario)) {
 647             timeout = 240
 648         }
 649         else if (isLongGc(scenario)) {
 650             timeout = 1440
 651         }
 652         else if (isJitDiff(scenario)) {
 653             timeout = 240
 654         }
 655         else if (isGcReliabilityFramework(scenario)) {
 656             timeout = 1440
 657         }
 658         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 659             timeout = 240
 660         }
 661     }
 662
 663     if (configuration == 'Debug') {
 664         // Debug runs can be very slow. Add an hour.
 665         timeout += 60
 666     }
 667
 668     // If we've changed the timeout from the default, set it in the job.
 669
 670     if (timeout != 120) {
 671         Utilities.setJobTimeout(newJob, timeout)
 672     }
 673 }
 674
 675 def static getJobFolder(def scenario) {
 676     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 677         return 'jitstress'
 678     }
 679     if (scenario == 'illink') {
 680         return 'illink'
 681     }
 682     return ''
 683 }
 684
 685 def static getStressModeDisplayName(def scenario) {
 686     def displayStr = ''
 687     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 688         def prefixLength = 'COMPlus_'.length()
 689         if (k.length() >= prefixLength) {
 690             def modeName = k.substring(prefixLength, k.length())
 691             displayStr += ' ' + modeName + '=' + v
 692         }
 693     }
 694
 695     if (isCoreFxScenario(scenario)) {
 696         displayStr = ('CoreFx ' + displayStr).trim()
 697     }
 698
 699     return displayStr
 700 }
 701
 702 def static getR2RDisplayName(def scenario) {
 703     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 704     def displayStr = scenario
 705     def prefixLength = 'r2r_'.length()
 706     if (displayStr.length() >= prefixLength) {
 707         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 708     } else if (scenario == 'r2r') {
 709         displayStr = "R2R"
 710     }
 711     return displayStr
 712 }
 713
 714 //
 715 // Functions to create an environment script.
 716 //      envScriptCreate -- initialize the script (call first)
 717 //      envScriptFinalize -- finalize the script (call last)
 718 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 719 //      envScriptAppendExistingScript -- append an existing script to the generated script
 720 //
 721 // Each script returns a string of commands. Concatenate all the strings together before
 722 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 723 //
 724
 725 // Initialize the environment setting script.
 726 def static envScriptCreate(def os, def stepScriptLocation) {
 727     def stepScript = ''
 728     if (os == 'Windows_NT') {
 729         stepScript += "echo Creating TestEnv script\r\n"
 730         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 731
 732         // Create at least an empty script.
 733         stepScript += "echo. > ${stepScriptLocation}\r\n"
 734     }
 735     else {
 736         stepScript += "echo Creating environment setting script\n"
 737         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 738     }
 739
 740     return stepScript
 741 }
 742
 743 // Generates the string for setting stress mode variables.
 744 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 745     def stepScript = ''
 746     if (os == 'Windows_NT') {
 747         stressModeVars.each{ k, v ->
 748             // Write out what we are writing to the script file
 749             stepScript += "echo Setting ${k}=${v}\r\n"
 750             // Write out the set itself to the script file`
 751             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 752         }
 753     }
 754     else {
 755         stressModeVars.each{ k, v ->
 756             // Write out what we are writing to the script file
 757             stepScript += "echo Setting ${k}=${v}\n"
 758             // Write out the set itself to the script file`
 759             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 760         }
 761     }
 762
 763     return stepScript
 764 }
 765
 766 // Append an existing script to an environment script.
 767 // Returns string of commands to do this.
 768 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 769     assert (os == 'Windows_NT')
 770     def stepScript = ''
 771
 772     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 773     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 774
 775     return stepScript
 776 }
 777
 778 // Finalize an environment setting script.
 779 // Returns string of commands to do this.
 780 def static envScriptFinalize(def os, def stepScriptLocation) {
 781     def stepScript = ''
 782
 783     if (os == 'Windows_NT') {
 784         // Display the resulting script. This is useful when looking at the output log file.
 785         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 786         stepScript += "type ${stepScriptLocation}\r\n"
 787     }
 788     else {
 789         stepScript += "chmod +x ${stepScriptLocation}\n"
 790     }
 791
 792     return stepScript
 793 }
 794
 795 def static isNeedDocker(def architecture, def os, def isBuild) {
 796     if (isBuild) {
 797         if (architecture == 'x86' && os == 'Ubuntu') {
 798             return true
 799         }
 800         else if (architecture == 'arm') {
 801             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 802                 return true
 803             }
 804         }
 805     }
 806     else {
 807         if (architecture == 'x86' && os == 'Ubuntu') {
 808             return true
 809         }
 810     }
 811     return false
 812 }
 813
 814 def static getDockerImageName(def architecture, def os, def isBuild) {
 815     // We must change some docker private images to official later
 816     if (isBuild) {
 817         if (architecture == 'x86' && os == 'Ubuntu') {
 818             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 819         }
 820         else if (architecture == 'arm') {
 821             if (os == 'Ubuntu') {
 822                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 823             }
 824             else if (os == 'Ubuntu16.04') {
 825                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 826             }
 827             else if (os == 'Tizen') {
 828                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 829             }
 830         }
 831     }
 832     else {
 833         if (architecture == 'x86' && os == 'Ubuntu') {
 834             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 835         }
 836     }
 837     println("Unknown architecture to use docker: ${architecture} ${os}");
 838     assert false
 839 }
 840
 841 // Calculates the name of the build job based on some typical parameters.
 842 //
 843 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 844     // If the architecture is x64, do not add that info into the build name.
 845     // Need to change around some systems and other builds to pick up the right builds
 846     // to do that.
 847
 848     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 849     if (isBuildOnly) {
 850         suffix += '_bld'
 851     }
 852     def baseName = ''
 853     switch (architecture) {
 854         case 'x64':
 855             if (scenario == 'normal') {
 856                 // For now we leave x64 off of the name for compatibility with other jobs
 857                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 858             }
 859             else if (scenario == 'formatting') {
 860                 // we don't care about the configuration for the formatting job. It runs all configs
 861                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 862             }
 863             else {
 864                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 865             }
 866             break
 867         case 'arm64':
 868             if (os.toLowerCase() == "windows_nt") {
 869                 // These are cross builds
 870                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 871             }
 872             else {
 873                 // Defaults to a small page size set of machines.
 874                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 875             }
 876             break
 877         case 'arm':
 878             // These are cross builds
 879             if (os == 'Tizen') {
 880                 // ABI: softfp
 881                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 882             }
 883             else {
 884                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 885             }
 886             break
 887         case 'armlb':
 888             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 889             break
 890         case 'x86':
 891         case 'x86_arm_altjit':
 892         case 'x64_arm64_altjit':
 893             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 894             break
 895         default:
 896             println("Unknown architecture: ${architecture}");
 897             assert false
 898             break
 899     }
 900
 901     return baseName + suffix
 902 }
 903
 904 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 905
 906     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 907     if (os == 'Windows_NT') {
 908         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 909             if (!(branch in Constants.WindowsArm64Branches)) {
 910                 return
 911             }
 912         }
 913     }
 914
 915     // Check scenario.
 916     switch (scenario) {
 917         case 'innerloop':
 918             break
 919         case 'normal':
 920             switch (architecture) {
 921                 case 'x64':
 922                 case 'x86':
 923                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 924                         addPeriodicTriggerHelper(job, '@daily')
 925                     }
 926                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 927                         addGithubPushTriggerHelper(job)
 928                     }
 929                     break
 930                 case 'arm':
 931                 case 'armlb':
 932                 case 'x86_arm_altjit':
 933                 case 'x64_arm64_altjit':
 934                     addGithubPushTriggerHelper(job)
 935                     break
 936                 case 'arm64':
 937                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 938                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 939                     break
 940                 default:
 941                     println("Unknown architecture: ${architecture}");
 942                     assert false
 943                     break
 944             }
 945             break
 946         case 'r2r':
 947             assert !(os in bidailyCrossList)
 948             // r2r gets a push trigger for checked/release
 949             if (configuration == 'Checked' || configuration == 'Release') {
 950                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 951                 if (architecture == 'x64' && os != 'OSX10.12') {
 952                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 953                     if (isFlowJob || os == 'Windows_NT') {
 954                         addGithubPushTriggerHelper(job)
 955                     }
 956                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 957                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 958                     if (isFlowJob) {
 959                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 960                     }
 961                 }
 962                 // For x86, only add per-commit jobs for Windows
 963                 else if (architecture == 'x86') {
 964                     if (os == 'Windows_NT') {
 965                         addGithubPushTriggerHelper(job)
 966                     }
 967                 }
 968                 // arm64 r2r jobs should only run daily.
 969                 else if (architecture == 'arm64') {
 970                     if (os == 'Windows_NT') {
 971                         addPeriodicTriggerHelper(job, '@daily')
 972                     }
 973                 }
 974             }
 975             break
 976         case 'r2r_jitstress1':
 977         case 'r2r_jitstress2':
 978         case 'r2r_jitstressregs1':
 979         case 'r2r_jitstressregs2':
 980         case 'r2r_jitstressregs3':
 981         case 'r2r_jitstressregs4':
 982         case 'r2r_jitstressregs8':
 983         case 'r2r_jitstressregs0x10':
 984         case 'r2r_jitstressregs0x80':
 985         case 'r2r_jitstressregs0x1000':
 986         case 'r2r_jitminopts':
 987         case 'r2r_jitforcerelocs':
 988         case 'r2r_gcstress15':
 989             assert !(os in bidailyCrossList)
 990
 991             // GCStress=C is currently not supported on OS X
 992             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 993                 break
 994             }
 995
 996             // GC Stress 15 r2r gets a push trigger for checked/release
 997             if (configuration == 'Checked' || configuration == 'Release') {
 998                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 999                 if (architecture == 'x64') {
1000                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1001                     if (isFlowJob || os == 'Windows_NT') {
1002                         // Add a weekly periodic trigger
1003                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1004                     }
1005                 }
1006                 // For x86, only add per-commit jobs for Windows
1007                 else if (architecture == 'x86') {
1008                     if (os == 'Windows_NT') {
1009                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1010                     }
1011                 }
1012             }
1013             break
1014         case 'longgc':
1015             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1016             assert configuration == 'Release'
1017             assert architecture == 'x64'
1018             addPeriodicTriggerHelper(job, '@daily')
1019             // TODO: Add once external email sending is available again
1020             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1021             break
1022         case 'gcsimulator':
1023             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1024             assert configuration == 'Release'
1025             assert architecture == 'x64'
1026             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1027             // TODO: Add once external email sending is available again
1028             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1029             break
1030         case 'standalone_gc':
1031             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1032             assert (configuration == 'Release' || configuration == 'Checked')
1033             // TODO: Add once external email sending is available again
1034             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1035             addPeriodicTriggerHelper(job, '@daily')
1036             break
1037         case 'gc_reliability_framework':
1038             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1039             assert (configuration == 'Release' || configuration == 'Checked')
1040             // Only triggered by phrase.
1041             break
1042         case 'ilrt':
1043             assert !(os in bidailyCrossList)
1044             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1045             if (architecture == 'x64' && configuration == 'Release') {
1046                 // We don't expect to see a job generated except in these scenarios
1047                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1048                 if (isFlowJob || os == 'Windows_NT') {
1049                     addPeriodicTriggerHelper(job, '@daily')
1050                 }
1051             }
1052             break
1053         case 'jitdiff':
1054             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1055             assert configuration == 'Checked'
1056             assert (architecture == 'x64' || architecture == 'x86')
1057             addGithubPushTriggerHelper(job)
1058             break
1059         case 'formatting':
1060             assert (os == 'Windows_NT' || os == "Ubuntu")
1061             assert architecture == 'x64'
1062             addGithubPushTriggerHelper(job)
1063             break
1064         case 'jitstressregs1':
1065         case 'jitstressregs2':
1066         case 'jitstressregs3':
1067         case 'jitstressregs4':
1068         case 'jitstressregs8':
1069         case 'jitstressregs0x10':
1070         case 'jitstressregs0x80':
1071         case 'jitstressregs0x1000':
1072         case 'minopts':
1073         case 'forcerelocs':
1074         case 'jitstress1':
1075         case 'jitstress2':
1076         case 'jitstress2_jitstressregs1':
1077         case 'jitstress2_jitstressregs2':
1078         case 'jitstress2_jitstressregs3':
1079         case 'jitstress2_jitstressregs4':
1080         case 'jitstress2_jitstressregs8':
1081         case 'jitstress2_jitstressregs0x10':
1082         case 'jitstress2_jitstressregs0x80':
1083         case 'jitstress2_jitstressregs0x1000':
1084         case 'tailcallstress':
1085         case 'jitsse2only':
1086         case 'jitnosimd':
1087         case 'jitnox86hwintrinsic':
1088         case 'jitincompletehwintrinsic':
1089         case 'jitx86hwintrinsicnoavx':
1090         case 'jitx86hwintrinsicnoavx2':
1091         case 'jitx86hwintrinsicnosimd':
1092         case 'corefx_baseline':
1093         case 'corefx_minopts':
1094         case 'corefx_jitstress1':
1095         case 'corefx_jitstress2':
1096         case 'corefx_jitstressregs1':
1097         case 'corefx_jitstressregs2':
1098         case 'corefx_jitstressregs3':
1099         case 'corefx_jitstressregs4':
1100         case 'corefx_jitstressregs8':
1101         case 'corefx_jitstressregs0x10':
1102         case 'corefx_jitstressregs0x80':
1103         case 'corefx_jitstressregs0x1000':
1104         case 'zapdisable':
1105             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1106                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1107                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1108                     if (os == 'Windows_NT') {
1109                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1110                         addPeriodicTriggerHelper(job, '@weekly')
1111                     }
1112                 }
1113                 else {
1114                     addPeriodicTriggerHelper(job, '@daily')
1115                 }
1116             }
1117             break
1118         case 'heapverify1':
1119         case 'gcstress0x3':
1120             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1121                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1122                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1123                     if (os == 'Windows_NT') {
1124                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1125                         addPeriodicTriggerHelper(job, '@weekly')
1126                     }
1127                     // TODO: Add once external email sending is available again
1128                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1129                 }
1130                 else {
1131                     addPeriodicTriggerHelper(job, '@weekly')
1132                 }
1133             }
1134             break
1135         case 'gcstress0xc':
1136         case 'gcstress0xc_zapdisable':
1137         case 'gcstress0xc_zapdisable_jitstress2':
1138         case 'gcstress0xc_zapdisable_heapverify1':
1139         case 'gcstress0xc_jitstress1':
1140         case 'gcstress0xc_jitstress2':
1141         case 'gcstress0xc_minopts_heapverify1':
1142             // GCStress=C is currently not supported on OS X
1143             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1144                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1145                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1146                     if (os == 'Windows_NT') {
1147                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1148                         addPeriodicTriggerHelper(job, '@weekly')
1149                     }
1150                     // TODO: Add once external email sending is available again
1151                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1152                 }
1153                 else {
1154                     addPeriodicTriggerHelper(job, '@weekly')
1155                 }
1156             }
1157             break
1158
1159         case 'illink':
1160             // Testing on other operating systems TBD
1161             assert (os == 'Windows_NT' || os == 'Ubuntu')
1162             if (architecture == 'x64' || architecture == 'x86') {
1163                 if (configuration == 'Checked') {
1164                     addPeriodicTriggerHelper(job, '@daily')
1165                 }
1166             }
1167             break
1168
1169         case 'tieredcompilation':
1170         case 'corefx_tieredcompilation':
1171             // No periodic jobs just yet, still testing
1172             break
1173
1174         default:
1175             println("Unknown scenario: ${scenario}");
1176             assert false
1177             break
1178     }
1179     return
1180 }
1181
1182 // **************************
1183 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1184 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1185 // builds will do a couple extra steps.
1186 // **************************
1187
1188 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1189 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1190 // it's just used for internal builds
1191 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1192 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1193     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1194
1195     if (isWindowsBuildOnlyJob) {
1196         return
1197     }
1198
1199     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1200     // Non pull request builds.
1201     if (!isPR) {
1202         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1203         return
1204     }
1205
1206      def arm64Users = [
1207         'AndyAyersMS',
1208         'briansull',
1209         'BruceForstall',
1210         'CarolEidt',
1211         'cmckinsey',
1212         'erozenfeld',
1213         'janvorli',
1214         'jashook',
1215         'JosephTremoulet',
1216         'pgodeq',
1217         'pgavlin',
1218         'rartemev',
1219         'russellhadley',
1220         'RussKeldorph',
1221         'sandreenko',
1222         'sdmaclea',
1223         'swaroop-sridhar',
1224         'jkotas',
1225         'markwilkie',
1226         'weshaggard'
1227     ]
1228
1229     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1230     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1231     def osGroup = getOSGroup(os)
1232     switch (architecture) {
1233         case 'x64': // editor brace matching: {
1234             if (scenario == 'formatting') {
1235                 assert configuration == 'Checked'
1236                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1237                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1238                 }
1239
1240                 break
1241             }
1242
1243             switch (os) {
1244                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1245                 case 'Debian8.4':
1246                 case 'RHEL7.2':
1247                     if (scenario == 'innerloop') {
1248                         assert !isFlowJob
1249                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1250                     }
1251                     else if (scenario == 'normal') {
1252                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1253                     }
1254                     break
1255
1256                 case 'Ubuntu16.04':
1257                     assert !isFlowJob
1258                     assert scenario != 'innerloop'
1259                     // Distinguish with the other architectures (arm and x86)
1260                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1261                     break
1262
1263                 case 'Fedora24':
1264                 case 'Ubuntu16.10':
1265                     assert !isFlowJob
1266                     assert scenario != 'innerloop'
1267                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1268                     break
1269
1270                 case 'Ubuntu':
1271                     if (scenario == 'illink') {
1272                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1273                         break
1274                     }
1275                     // fall through
1276
1277                 case 'OSX10.12':
1278                     // Triggers on the non-flow jobs aren't necessary here
1279                     // Corefx testing uses non-flow jobs.
1280                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1281                         break
1282                     }
1283                     switch (scenario) {
1284                         case 'innerloop':
1285                             // PR Triggered jobs. These jobs will run pri0 tests.
1286                             if (configuration == 'Checked') {
1287                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1288                             }
1289                             break
1290
1291                         case 'normal':
1292                             // OSX uses checked for default PR tests
1293                             if (configuration == 'Checked') {
1294                                 // Default trigger
1295                                 assert !job.name.contains("centos")
1296                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1297                             }
1298                             break
1299
1300                         case 'jitdiff':
1301                             if (configuration == 'Checked') {
1302                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1303                             }
1304                             break
1305
1306                         case 'ilrt':
1307                             if (configuration == 'Release') {
1308                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1309                             }
1310                             break
1311
1312                         case 'longgc':
1313                             if (configuration == 'Release') {
1314                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1315                             }
1316                             break
1317
1318                         case 'gcsimulator':
1319                             if (configuration == 'Release') {
1320                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1321                             }
1322                             break
1323
1324                         case 'standalone_gc':
1325                             if (configuration == 'Release' || configuration == 'Checked') {
1326                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1327                             }
1328                             break
1329
1330                         case 'gc_reliability_framework':
1331                             if (configuration == 'Release' || configuration == 'Checked') {
1332                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1333                             }
1334                             break
1335
1336                         default:
1337                             if (isJitStressScenario(scenario)) {
1338                                 def displayStr = getStressModeDisplayName(scenario)
1339                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1340                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1341                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1342                             }
1343                             else if (isR2RScenario(scenario)) {
1344                                 if (configuration == 'Release' || configuration == 'Checked') {
1345                                     def displayStr = getR2RDisplayName(scenario)
1346                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1347                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1348                                 }
1349                             }
1350                             else {
1351                                 println("Unknown scenario: ${scenario}");
1352                                 assert false
1353                             }
1354                             break
1355
1356                     }
1357                     break
1358
1359                 case 'CentOS7.1':
1360                     switch (scenario) {
1361                         case 'innerloop':
1362                             // CentOS uses checked for default PR tests while debug is build only
1363                             if (configuration == 'Debug') {
1364                                 // Default trigger
1365                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1366                             }
1367
1368                             // Make sure this is a flow job to get build and test.
1369                             if (configuration == 'Checked' && isFlowJob) {
1370                                 assert job.name.contains("flow")
1371                                 // Default trigger
1372                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1373                             }
1374                             break
1375
1376                         case 'normal':
1377                             // Make sure this is a flow job to get build and test.
1378                             if (configuration == 'Checked' && isFlowJob) {
1379                                 assert job.name.contains("flow")
1380                                 // Default trigger
1381                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1382                             }
1383                             break
1384
1385                         default:
1386                             if (isR2RScenario(scenario)) {
1387                                 if (configuration == 'Release' || configuration == 'Checked') {
1388                                     def displayStr = getR2RDisplayName(scenario)
1389                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1390                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1391                                 }
1392                             }
1393                             break
1394
1395                     }
1396                     break
1397
1398                 case 'Windows_NT':
1399                     switch (scenario) {
1400                         case 'innerloop':
1401                             // Default trigger
1402                             if (configuration == 'Checked' || configuration == 'Release') {
1403                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1404                             }
1405                             break
1406
1407                         case 'normal':
1408                             if (configuration == 'Checked') {
1409                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1410                             }
1411                             break
1412
1413                         case 'jitdiff':
1414                             if (configuration == 'Checked') {
1415                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1416                             }
1417                             break
1418
1419                         case 'ilrt':
1420                             if (configuration == 'Release') {
1421                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1422                             }
1423                             break
1424
1425                         case 'longgc':
1426                             if (configuration == 'Release') {
1427                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1428                             }
1429                             break
1430
1431                         case 'gcsimulator':
1432                             if (configuration == 'Release') {
1433                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1434                             }
1435                             break
1436
1437                         case 'standalone_gc':
1438                             if (configuration == 'Release' || configuration == 'Checked') {
1439                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1440                             }
1441                             break
1442
1443                         case 'gc_reliability_framework':
1444                             if (configuration == 'Release' || configuration == 'Checked') {
1445                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1446                             }
1447                             break
1448
1449                         case 'illink':
1450                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1451                             break
1452
1453                         default:
1454                             if (isJitStressScenario(scenario)) {
1455                                 def displayStr = getStressModeDisplayName(scenario)
1456                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1457                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1458                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1459                             }
1460                             else if (isR2RScenario(scenario)) {
1461                                 if (configuration == 'Release' || configuration == 'Checked') {
1462                                     def displayStr = getR2RDisplayName(scenario)
1463                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1464                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1465                                 }
1466                             }
1467                             else {
1468                                 println("Unknown scenario: ${scenario}");
1469                                 assert false
1470                             }
1471                             break
1472
1473                     }
1474                     break
1475
1476                 default:
1477                     println("Unknown os: ${os}");
1478                     assert false
1479                     break
1480
1481             }
1482
1483             break
1484
1485         // editor brace matching: }
1486         case 'armlb':
1487         case 'arm': // editor brace matching: {
1488             switch (os) {
1489                 case 'Ubuntu':
1490                 case 'Ubuntu16.04':
1491                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1492                         break
1493                     }
1494
1495                     assert scenario != 'innerloop'
1496                     job.with {
1497                         publishers {
1498                             azureVMAgentPostBuildAction {
1499                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1500                             }
1501                         }
1502                     }
1503                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1504                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1505                     break
1506
1507                 case 'Tizen':
1508                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1509                         break
1510                     }
1511
1512                     architecture = 'armel'
1513                     job.with {
1514                         publishers {
1515                             azureVMAgentPostBuildAction {
1516                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1517                             }
1518                         }
1519                     }
1520
1521                     if (scenario == 'innerloop') {
1522                         if (configuration == 'Checked') {
1523                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1524                         }
1525                     }
1526                     else {
1527                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1528                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1529                     }
1530                     break
1531
1532                 case 'Windows_NT':
1533                     // Triggers on the non-flow jobs aren't necessary here
1534                     if (!isFlowJob) {
1535                         break
1536                     }
1537
1538                     // Set up a private trigger
1539                     def contextString = "${os} ${architecture} Cross ${configuration}"
1540                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1541                     if (scenario == 'innerloop') {
1542                         contextString += " Innerloop"
1543                         triggerString += "\\W+Innerloop"
1544                     }
1545                     else {
1546                         contextString += " ${scenario}"
1547                         triggerString += "\\W+${scenario}"
1548                     }
1549
1550                     if (configuration == 'Debug') {
1551                         contextString += " Build"
1552                         triggerString += "\\W+Build"
1553                     } else {
1554                         contextString += " Build and Test"
1555                         triggerString += "\\W+Build and Test"
1556                     }
1557
1558                     triggerString += ".*"
1559
1560                     switch (scenario) {
1561                         case 'innerloop':
1562                             // Only Checked is an innerloop trigger.
1563                             if (configuration == 'Checked')
1564                             {
1565                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1566                             }
1567                             break
1568                         case 'normal':
1569                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1570                             break
1571                         default:
1572                             // Stress jobs will use this code path.
1573                             if (isArmWindowsScenario(scenario)) {
1574                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1575                             }
1576                             break
1577                     }
1578                     break
1579                 default:
1580                     println("NYI os: ${os}");
1581                     assert false
1582                     break
1583             }
1584             break
1585         // editor brace matching: }
1586         case 'arm64': // editor brace matching: {
1587             // Set up a private trigger
1588             def contextString = "${os} ${architecture} Cross ${configuration}"
1589             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1590
1591             if (scenario == 'innerloop') {
1592                 contextString += " Innerloop"
1593                 triggerString += "\\W+Innerloop"
1594             }
1595             else {
1596                 contextString += " ${scenario}"
1597                 triggerString += "\\W+${scenario}"
1598             }
1599
1600             if (configuration == 'Debug') {
1601                 contextString += " Build"
1602                 triggerString += "\\W+Build"
1603             } else {
1604                 contextString += " Build and Test"
1605                 triggerString += "\\W+Build and Test"
1606             }
1607
1608             triggerString += ".*"
1609
1610             switch (os) {
1611                 case 'Ubuntu':
1612                 case 'Ubuntu16.04':
1613                     switch (scenario) {
1614                         case 'innerloop':
1615                             if (configuration == 'Debug' && !isFlowJob) {
1616                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1617                             }
1618
1619                             break
1620                         case 'normal':
1621                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1622                             break
1623                         default:
1624                             if (isR2RScenario(scenario)) {
1625                                 if (configuration == 'Checked' || configuration == 'Release') {
1626                                     def displayStr = getR2RDisplayName(scenario)
1627                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1628                                 }
1629                             }
1630                             break
1631                     }
1632                     break
1633
1634                 case 'Windows_NT':
1635                     // Triggers on the non-flow jobs aren't necessary here
1636                     if (!isFlowJob) {
1637                         break
1638                     }
1639
1640                     assert isArmWindowsScenario(scenario)
1641                     switch (scenario) {
1642                         case 'innerloop':
1643                             if (configuration == 'Checked') {
1644                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1645                             }
1646
1647                             break
1648                         case 'normal':
1649                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1650                             break
1651                         default:
1652                             // Stress jobs will use this code path.
1653                             if (isArmWindowsScenario(scenario)) {
1654                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1655                             }
1656                             break
1657                     }
1658                     break
1659                 default:
1660                     println("NYI os: ${os}");
1661                     assert false
1662                     break
1663             }
1664             break
1665
1666         // editor brace matching: }
1667         case 'x86': // editor brace matching: {
1668             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1669             if (os == 'Ubuntu') {
1670                 // Triggers on the non-flow jobs aren't necessary here
1671                 if (!isFlowJob) {
1672                     break
1673                 }
1674
1675                 // on-demand only for ubuntu x86
1676                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1677                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1678                 break
1679
1680             }
1681             switch (scenario) {
1682                 case 'innerloop':
1683                     if (configuration == 'Checked' || configuration == 'Release') {
1684                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1685                     }
1686                     break
1687
1688                 case 'normal':
1689                     if (configuration == 'Checked') {
1690                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1691                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1692                     }
1693                     break
1694
1695                 case 'ilrt':
1696                     if (configuration == 'Release') {
1697                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1698                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1699                     }
1700                     break
1701
1702                 case 'longgc':
1703                     if (configuration == 'Release') {
1704                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1705                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1706                     }
1707                     break
1708
1709                 case 'gcsimulator':
1710                     if (configuration == 'Release') {
1711                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1712                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1713                     }
1714                     break
1715
1716                 case 'standalone_gc':
1717                     if (configuration == 'Release' || configuration == 'Checked') {
1718                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1719                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1720                     }
1721                     break
1722
1723                 case 'illink':
1724                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1725                     break
1726
1727                 default:
1728                     if (isJitStressScenario(scenario)) {
1729                         def displayStr = getStressModeDisplayName(scenario)
1730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1731                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1732                     }
1733                     else if (isR2RScenario(scenario)) {
1734                         if (configuration == 'Release' || configuration == 'Checked') {
1735                             def displayStr = getR2RDisplayName(scenario)
1736                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1737                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1738                         }
1739                     }
1740                     else {
1741                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1742                         assert false
1743                     }
1744                     break
1745
1746             }
1747             break
1748
1749          // editor brace matching: }
1750         case 'x64_arm64_altjit':
1751         case 'x86_arm_altjit': // editor brace matching: {
1752             assert (os == 'Windows_NT')
1753             switch (scenario) {
1754                 case 'normal':
1755                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1756                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1757                     break
1758                 default:
1759                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1760                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1761                     break
1762             }
1763             break
1764
1765         // editor brace matching: }
1766         default:
1767             println("Unknown architecture: ${architecture}");
1768             assert false
1769             break
1770     }
1771 }
1772
1773 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1774     def buildCommands = [];
1775     def osGroup = getOSGroup(os)
1776     def lowerConfiguration = configuration.toLowerCase()
1777
1778     def priority = '1'
1779     if (scenario == 'innerloop') {
1780         priority = '0'
1781     }
1782
1783     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1784
1785     def enableCorefxTesting = isCoreFxScenario(scenario)
1786
1787     // Calculate the build steps, archival, and xunit results
1788     switch (os) {
1789         case 'Windows_NT': // editor brace matching: {
1790             switch (architecture) {
1791                 case 'x64':
1792                 case 'x86':
1793                 case 'x86_arm_altjit':
1794                 case 'x64_arm64_altjit':
1795                     def arch = architecture
1796                     def buildOpts = ''
1797                     if (architecture == 'x86_arm_altjit') {
1798                         arch = 'x86'
1799                     }
1800                     else if (architecture == 'x64_arm64_altjit') {
1801                         arch = 'x64'
1802                     }
1803
1804                     if (scenario == 'formatting') {
1805                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1806                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1807                         break
1808                     }
1809
1810                     if (scenario == 'illink') {
1811                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1812                     }
1813
1814                     // If it is a release build for windows, ensure PGO is used, else fail the build
1815                     if ((lowerConfiguration == 'release') &&
1816                         (scenario in Constants.basicScenarios) &&
1817                         (architecture != 'x86_arm_altjit') &&
1818                         (architecture != 'x64_arm64_altjit')) {
1819
1820                         buildOpts += ' -enforcepgo'
1821                     }
1822
1823                     if (enableCorefxTesting) {
1824                         buildOpts += ' skiptests';
1825                     } else {
1826                         buildOpts += " -priority=${priority}"
1827                     }
1828
1829                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1830                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1831                     // 35 characters long.
1832
1833                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1834
1835                     if (!isBuildOnly) {
1836                         def runtestArguments = ''
1837                         def testOpts = 'collectdumps'
1838
1839                         if (isR2RScenario(scenario)) {
1840
1841                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1842                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1843                             // to cause the tests to be crossgen'ed.
1844
1845                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1846                                 testOpts += ' crossgenaltjit protononjit.dll'
1847                             } else {
1848                                 testOpts += ' crossgen'
1849                             }
1850
1851                             testOpts += ' runcrossgentests'
1852                         }
1853                         else if (scenario == 'jitdiff') {
1854                             testOpts += ' jitdisasm crossgen'
1855                         }
1856                         else if (scenario == 'ilrt') {
1857                             testOpts += ' ilasmroundtrip'
1858                         }
1859                         else if (isLongGc(scenario)) {
1860                             testOpts += " ${scenario} sequential"
1861                         }
1862                         else if (scenario == 'standalone_gc') {
1863                             testOpts += ' gcname clrgc.dll'
1864                         }
1865                         else if (scenario == 'illink') {
1866                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1867                         }
1868
1869                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1870                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1871                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1872                         // which is read by the xunit harness.)
1873                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1874                         {
1875                             def timeout = 1800000
1876                             testOpts += " timeout ${timeout}"
1877                         }
1878
1879                         // If we are running a stress mode, we should write out the set of key
1880                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1881
1882                         def envScriptPath = ''
1883                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1884                             def buildCommandsStr = ''
1885                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1886                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1887
1888                             if (isJitStressScenario(scenario)) {
1889                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1890                             }
1891                             else if (isR2RStressScenario(scenario)) {
1892                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1893                             }
1894
1895                             if (architecture == 'x86_arm_altjit') {
1896                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1897                             }
1898                             else if (architecture == 'x64_arm64_altjit') {
1899                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1900                             }
1901
1902                             envScriptFinalize(os, envScriptPath)
1903
1904                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1905                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1906                             // into a single script.
1907                             buildCommands += buildCommandsStr
1908                         }
1909                         else if (architecture == 'x86_arm_altjit') {
1910                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1911                         }
1912                         else if (architecture == 'x64_arm64_altjit') {
1913                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1914                         }
1915                         if (envScriptPath != '') {
1916                             testOpts += " TestEnv ${envScriptPath}"
1917                         }
1918
1919                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1920
1921                         if (enableCorefxTesting) {
1922                             def workspaceRelativeFxRoot = "_/fx"
1923                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1924
1925                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1926
1927                             // Archive and process (only) the test results
1928                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1929                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1930
1931                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1932                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1933                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1934                         }
1935                         else if (isGcReliabilityFramework(scenario)) {
1936                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1937                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1938                         }
1939                         else {
1940                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1941                         }
1942                     }
1943
1944                     if (!enableCorefxTesting) {
1945                         // Run the rest of the build
1946                         // Build the mscorlib for the other OS's
1947                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1948                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1949
1950                         if (arch == "x64") {
1951                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1952                         }
1953
1954                         // Zip up the tests directory so that we don't use so much space/time copying
1955                         // 10s of thousands of files around.
1956                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1957
1958                         if (!isJitStressScenario(scenario)) {
1959                             // For Windows, pull full test results and test drops for x86/x64.
1960                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1961                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1962                         }
1963
1964                         if (scenario == 'jitdiff') {
1965                             // retrieve jit-dasm output for base commit, and run jit-diff
1966                             if (!isBuildOnly) {
1967                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1968                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1969                             }
1970                         }
1971
1972                         if (!isBuildOnly) {
1973                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1974                         }
1975                     }
1976                     break
1977                 case 'armlb':
1978                 case 'arm':
1979                     assert isArmWindowsScenario(scenario)
1980
1981                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1982                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1983
1984                     def buildArchitecture = 'arm'
1985
1986                     def buildOpts = ''
1987
1988                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1989                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1990                     if (architecture == 'armlb') {
1991                         buildOpts += ' -crossgenaltjit legacyjit.dll'
1992                     }
1993
1994                     if (enableCorefxTesting) {
1995                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
1996                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
1997                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
1998                         //
1999                         // buildOpts += ' skiptests';
2000                         buildOpts += " -priority=0"
2001                     } else {
2002                         buildOpts += " -priority=${priority}"
2003                     }
2004
2005                     // This is now a build only job. Do not run tests. Use the flow job.
2006                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2007
2008                     if (enableCorefxTesting) {
2009                         assert isBuildOnly
2010                         assert architecture == 'arm'
2011
2012                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2013                         // to determine the correct matching corefx version git commit hash.
2014                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2015
2016                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2017                         def envScriptPath = ''
2018                         def buildCommandsStr = ''
2019                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2020                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2021                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2022                         envScriptFinalize(os, envScriptPath)
2023                         buildCommands += buildCommandsStr
2024
2025                         def workspaceRelativeFxRootLinux = "_/fx"
2026                         def workspaceRelativeFxRootWin = "_\\fx"
2027                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2028
2029                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2030
2031                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2032                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2033                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2034
2035                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2036                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2037                     } else {
2038                         // Zip up the tests directory so that we don't use so much space/time copying
2039                         // 10s of thousands of files around.
2040                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2041
2042                         // Add archival.
2043                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2044                     }
2045                     break
2046                 case 'arm64':
2047                     assert isArmWindowsScenario(scenario)
2048
2049                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2050                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2051
2052                     // This is now a build only job. Do not run tests. Use the flow job.
2053                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2054
2055                     // Zip up the tests directory so that we don't use so much space/time copying
2056                     // 10s of thousands of files around.
2057                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2058
2059                     // Add archival.
2060                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2061                     break
2062                 default:
2063                     println("Unknown architecture: ${architecture}");
2064                     assert false
2065                     break
2066             }
2067             break
2068         // editor brace matching: }
2069         case 'Ubuntu':
2070         case 'Ubuntu16.04':
2071         case 'Ubuntu16.10':
2072         case 'Debian8.4':
2073         case 'OSX10.12':
2074         case 'CentOS7.1':
2075         case 'RHEL7.2':
2076         case 'Tizen':
2077         case 'Fedora24': // editor brace matching: {
2078             switch (architecture) {
2079                 case 'x64':
2080                 case 'x86':
2081                     if (architecture == 'x86' && os == 'Ubuntu') {
2082                         // build and PAL test
2083                         def dockerImage = getDockerImageName(architecture, os, true)
2084                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2085                         dockerImage = getDockerImageName(architecture, os, false)
2086                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2087                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2088                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2089                         break
2090                     }
2091
2092                     if (scenario == 'formatting') {
2093                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2094                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2095                         break
2096                     }
2097
2098                     if (scenario == 'illink') {
2099                         assert(os == 'Ubuntu')
2100                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2101                     }
2102
2103                     if (!enableCorefxTesting) {
2104                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2105                         // only on supported OS platforms.
2106                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2107                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2108
2109                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2110                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2111
2112                         // Basic archiving of the build
2113                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2114                         // And pal tests
2115                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2116                     }
2117                     else {
2118                         // Corefx stress testing
2119                         assert os == 'Ubuntu'
2120                         assert architecture == 'x64'
2121                         assert lowerConfiguration == 'checked'
2122                         assert isJitStressScenario(scenario)
2123
2124                         // Build coreclr
2125                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2126
2127                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2128
2129                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2130                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2131                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2132                         buildCommands += envScriptCmds
2133
2134                         // Build and text corefx
2135                         def workspaceRelativeFxRoot = "_/fx"
2136                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2137
2138                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2139
2140                         // Archive and process (only) the test results
2141                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2142                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2143                     }
2144                     break
2145                 case 'arm64':
2146                     if (!enableCorefxTesting) {
2147                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2148
2149                         // HACK -- Arm64 does not have corefx jobs yet.
2150                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2151                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2152                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2153                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2154
2155                         // Basic archiving of the build
2156                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2157                     }
2158                     break
2159                 case 'arm':
2160                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2161                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2162
2163                     // default values for Ubuntu
2164                     def arm_abi="arm"
2165                     def linuxCodeName="trusty"
2166                     if (os == 'Ubuntu16.04') {
2167                         linuxCodeName="xenial"
2168                     }
2169                     else if (os == 'Tizen') {
2170                         arm_abi="armel"
2171                         linuxCodeName="tizen"
2172                     }
2173
2174                     // Unzip the Windows test binaries first. Exit with 0
2175                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2176
2177                     // Unpack the corefx binaries
2178                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2179                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2180                     if (os != 'Tizen') {
2181                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2182                     }
2183                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2184                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2185                     if (os == 'Ubuntu') {
2186                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2187                     }
2188
2189                     // Call the ARM CI script to cross build and test using docker
2190                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2191                     --mode=docker \\
2192                     --${arm_abi} \\
2193                     --linuxCodeName=${linuxCodeName} \\
2194                     --buildConfig=${lowerConfiguration} \\
2195                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2196                     --coreFxBinDir=./bin/CoreFxBinDir \\
2197                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2198
2199                     // Basic archiving of the build, no pal tests
2200                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2201                     break
2202                 default:
2203                     println("Unknown architecture: ${architecture}");
2204                     assert false
2205                     break
2206             }
2207             break
2208         // editor brace matching: }
2209         default:
2210             println("Unknown os: ${os}");
2211             assert false
2212             break
2213     } // os
2214
2215     return buildCommands
2216 }
2217
2218 Constants.allScenarios.each { scenario ->
2219     [true, false].each { isPR ->
2220         Constants.architectureList.each { architecture ->
2221             Constants.configurationList.each { configuration ->
2222                 Constants.osList.each { os ->
2223                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2224                     // and reset the os to Windows_NT
2225                     def isBuildOnly = false
2226                     if (os == 'Windows_NT_BuildOnly') {
2227                         isBuildOnly = true
2228                         os = 'Windows_NT'
2229                     }
2230
2231                     // Tizen is only supported for arm architecture
2232                     if (os == 'Tizen' && architecture != 'arm') {
2233                         return
2234                     }
2235
2236                     // Skip totally unimplemented (in CI) configurations.
2237                     switch (architecture) {
2238                         case 'arm64':
2239                             if (os == 'Ubuntu16.04') {
2240                                 os = 'Ubuntu'
2241                             }
2242
2243                             // Windows and Ubuntu only
2244                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2245                                 return
2246                             }
2247                             break
2248                         case 'arm':
2249                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2250                                 return
2251                             }
2252                             break
2253                         case 'armlb':
2254                             if (os != 'Windows_NT') {
2255                                 return
2256                             }
2257                             break
2258                         case 'x86':
2259                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2260                                 return
2261                             }
2262                             break
2263                         case 'x86_arm_altjit':
2264                         case 'x64_arm64_altjit':
2265                             if (os != 'Windows_NT') {
2266                                 return
2267                             }
2268                             break
2269                         case 'x64':
2270                             // Everything implemented
2271                             break
2272                         default:
2273                             println("Unknown architecture: ${architecture}")
2274                             assert false
2275                             break
2276                     }
2277
2278                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2279                     // with checked builds)
2280                     if (isJitStressScenario(scenario)) {
2281                         if (configuration != 'Checked') {
2282                             return
2283                         }
2284
2285                         // Since these are just execution time differences,
2286                         // skip platforms that don't execute the tests here (Windows_NT only)
2287                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2288                         if (!isEnabledOS) {
2289                             return
2290                         }
2291
2292                         switch (architecture) {
2293                             case 'x64':
2294                             case 'x86':
2295                             case 'x86_arm_altjit':
2296                             case 'x64_arm64_altjit':
2297                                 // x86 ubuntu: default only
2298                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2299                                     return
2300                                 }
2301                                 if (isBuildOnly) {
2302                                     return
2303                                 }
2304                                 break
2305
2306                             case 'arm':
2307                                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2308                                 if (!isBuildOnly || !isCoreFxScenario(scenario)) {
2309                                     return
2310                                 }
2311                                 break
2312
2313                             default:
2314                                 // arm64, armlb: stress is handled through flow jobs.
2315                                 return
2316                         }
2317                     }
2318                     else if (isR2RScenario(scenario)) {
2319                         if (os != 'Windows_NT') {
2320                             return
2321                         }
2322                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2323                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2324                             return
2325                         }
2326                     }
2327                     else {
2328                         // Skip scenarios
2329                         switch (scenario) {
2330                             case 'ilrt':
2331                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2332                                 // the default scenario build
2333                                 if (os != 'Windows_NT') {
2334                                     return
2335                                 }
2336                                 // Only x64 for now
2337                                 if (architecture != 'x64') {
2338                                     return
2339                                 }
2340                                 // Release only
2341                                 if (configuration != 'Release') {
2342                                     return
2343                                 }
2344                                 break
2345                             case 'jitdiff':
2346                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2347                                     return
2348                                 }
2349                                 if (architecture != 'x64') {
2350                                     return
2351                                 }
2352                                 if (configuration != 'Checked') {
2353                                     return
2354                                 }
2355                                 break
2356                             case 'longgc':
2357                             case 'gcsimulator':
2358                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2359                                     return
2360                                 }
2361                                 if (architecture != 'x64') {
2362                                     return
2363                                 }
2364                                 if (configuration != 'Release') {
2365                                     return
2366                                 }
2367                                 break
2368                             case 'gc_reliability_framework':
2369                             case 'standalone_gc':
2370                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2371                                     return
2372                                 }
2373
2374                                 if (architecture != 'x64') {
2375                                     return
2376                                 }
2377
2378                                 if (configuration != 'Release' && configuration != 'Checked') {
2379                                     return
2380                                 }
2381                                 break
2382                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2383                             case 'formatting':
2384                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2385                                     return
2386                                 }
2387                                 if (architecture != 'x64') {
2388                                     return
2389                                 }
2390                                 if (configuration != 'Checked') {
2391                                     return
2392                                 }
2393                                 if (isBuildOnly) {
2394                                     return
2395                                 }
2396                                 break
2397                             case 'illink':
2398                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2399                                     return
2400                                 }
2401                                 if (architecture != 'x64' && architecture != 'x86') {
2402                                     return
2403                                 }
2404                                 if (isBuildOnly) {
2405                                     return
2406                                 }
2407                                 break
2408                             case 'normal':
2409                                 // Nothing skipped
2410                                 break
2411                             case 'innerloop':
2412                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2413                                     return
2414                                 }
2415                                 break
2416                             default:
2417                                 println("Unknown scenario: ${scenario}")
2418                                 assert false
2419                                 break
2420                         }
2421                     }
2422
2423                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2424                     // runtime behavior, not compile-time behavior, are not interesting.
2425                     switch (architecture) {
2426                         case 'x86_arm_altjit':
2427                         case 'x64_arm64_altjit':
2428                             if (isGCStressRelatedTesting(scenario)) {
2429                                 return
2430                             }
2431                             break
2432                         default:
2433                             break
2434                     }
2435
2436                     // Calculate names
2437                     def lowerConfiguration = configuration.toLowerCase()
2438                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2439                     def folderName = getJobFolder(scenario)
2440
2441                     // Create the new job
2442                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2443                     addToViews(newJob, isPR, architecture, os)
2444
2445                     def machineAffinityOptions = null
2446
2447                     if (os != 'Windows_NT') {
2448                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2449                     }
2450                     else {
2451                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2452                     }
2453
2454                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2455
2456                     // Add all the standard options
2457                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2458                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2459
2460                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2461                     def osGroup = getOSGroup(os)
2462
2463                     newJob.with {
2464                         steps {
2465                             if (os == 'Windows_NT') {
2466                                 buildCommands.each { buildCommand ->
2467                                     batchFile(buildCommand)
2468                                 }
2469                             }
2470                             else {
2471                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2472                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2473                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2474                                     // Define the Windows Tests and Corefx build job names
2475                                     def WindowsTestsName = projectFolder + '/' +
2476                                                            Utilities.getFullJobName(project,
2477                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2478                                                                                     false)
2479                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2480                                                        Utilities.getFolderName(branch)
2481
2482                                     // Copy the Windows test binaries and the Corefx build binaries
2483                                     copyArtifacts(WindowsTestsName) {
2484                                         includePatterns('bin/tests/tests.zip')
2485                                         buildSelector {
2486                                             latestSuccessful(true)
2487                                         }
2488                                     }
2489
2490                                     def arm_abi = 'arm'
2491                                     def corefx_os = 'linux'
2492                                     if (os == 'Tizen') {
2493                                         arm_abi = 'armel'
2494                                         corefx_os = 'tizen'
2495                                     }
2496
2497                                     // Let's use release CoreFX to test checked CoreCLR,
2498                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2499                                     def corefx_lowerConfiguration = lowerConfiguration
2500                                     if ( lowerConfiguration == 'checked' ) {
2501                                         corefx_lowerConfiguration='release'
2502                                     }
2503
2504                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2505                                         includePatterns('bin/build.tar.gz')
2506                                         buildSelector {
2507                                             latestSuccessful(true)
2508                                         }
2509                                     }
2510                                 }
2511
2512                                 buildCommands.each { buildCommand ->
2513                                     shell(buildCommand)
2514                                 }
2515                             }
2516                         }
2517                     } // newJob.with
2518
2519                 } // os
2520             } // configuration
2521         } // architecture
2522     } // isPR
2523 } // scenario
2524
2525
2526 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2527 Constants.allScenarios.each { scenario ->
2528     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2529
2530     [true, false].each { isPR ->
2531         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2532             Constants.crossList.each { os ->
2533                 if (architecture == 'arm64') {
2534                     if (os != "Ubuntu" && os != "Windows_NT") {
2535                         return
2536                     }
2537                 } else if (architecture == 'arm' || architecture == 'armlb') {
2538                     if (os != 'Windows_NT') {
2539                         return
2540                     }
2541                 } else if (architecture == 'x86') {
2542                     if (os != "Ubuntu") {
2543                         return
2544                     }
2545                 }
2546
2547                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2548
2549                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2550                     return
2551                 }
2552
2553                 Constants.configurationList.each { configuration ->
2554
2555                     // First, filter based on OS.
2556
2557                     if (os == 'Windows_NT') {
2558                         if (!isArmWindowsScenario(scenario)) {
2559                             return
2560                         }
2561                     }
2562                     else {
2563                         // Non-Windows
2564                         if (architecture == 'arm64') {
2565                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2566                                 return
2567                             }
2568                         }
2569                         else if (architecture == 'x86') {
2570                             // Linux/x86 only want innerloop and default test
2571                             if (!isNormalOrInnerloop) {
2572                                 return
2573                             }
2574                         }
2575                     }
2576
2577                     // For CentOS, we only want Checked/Release builds.
2578                     if (os == 'CentOS7.1') {
2579                         if (configuration != 'Checked' && configuration != 'Release') {
2580                             return
2581                         }
2582                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2583                             return
2584                         }
2585                     }
2586
2587                     // For RedHat and Debian, we only do Release builds.
2588                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2589                         if (configuration != 'Release') {
2590                             return
2591                         }
2592                         if (!isNormalOrInnerloop) {
2593                             return
2594                         }
2595                     }
2596
2597                     // Next, filter based on scenario.
2598
2599                     if (isJitStressScenario(scenario)) {
2600                         if (configuration != 'Checked') {
2601                             return
2602                         }
2603
2604                         // CoreFx JIT stress tests currently only implemented for ARM.
2605                         if (isCoreFxScenario(scenario) && (architecture != 'arm')) {
2606                             return
2607                         }
2608                     }
2609                     else if (isR2RBaselineScenario(scenario)) {
2610                         if (configuration != 'Checked' && configuration != 'Release') {
2611                             return
2612                         }
2613                     }
2614                     else if (isR2RStressScenario(scenario)) {
2615                         if (configuration != 'Checked') {
2616                             return
2617                         }
2618                     }
2619                     else {
2620                         // Skip scenarios
2621                         switch (scenario) {
2622                             case 'ilrt':
2623                             case 'longgc':
2624                             case 'gcsimulator':
2625                                 // Long GC tests take a long time on non-Release builds
2626                                 // ilrt is also Release only
2627                                 if (configuration != 'Release') {
2628                                     return
2629                                 }
2630                                 break
2631
2632                             case 'jitdiff':
2633                                 if (configuration != 'Checked') {
2634                                     return;
2635                                 }
2636                                 break
2637
2638                             case 'gc_reliability_framework':
2639                             case 'standalone_gc':
2640                                 if (configuration != 'Release' && configuration != 'Checked') {
2641                                     return
2642                                 }
2643                                 break
2644
2645                             case 'formatting':
2646                                 return
2647                             case 'illink':
2648                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2649                                     return
2650                                 }
2651                                 break
2652
2653                             case 'normal':
2654                                 // Nothing skipped
2655                                 break
2656
2657                             case 'innerloop':
2658                                 // Nothing skipped
2659                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2660                                     return
2661                                 }
2662                                 break
2663
2664                             default:
2665                                 println("Unknown scenario: ${scenario}")
2666                                 assert false
2667                                 break
2668                         }
2669                     }
2670
2671                     // Done filtering. Now, create the jobs.
2672
2673                     // =============================================================================================
2674                     // Create the test job
2675                     // =============================================================================================
2676
2677                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2678
2679                     def lowerConfiguration = configuration.toLowerCase()
2680                     def osGroup = getOSGroup(os)
2681                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2682
2683                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2684                     def inputCoreCLRBuildIsBuildOnly = false
2685                     if (isCoreFxScenario(scenario)) {
2686                         // Every CoreFx test depends on its own unique build.
2687                         inputCoreCLRBuildScenario = scenario
2688                         inputCoreCLRBuildIsBuildOnly = true
2689                     }
2690                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
2691                     def inputCoreCLRBuildName = projectFolder + '/' +
2692                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
2693
2694                     def inputWindowsTestsBuildName = ""
2695                     if (windowsArmJob != true) {
2696                         // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2697                         // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2698
2699                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2700
2701                         def inputWindowsTestBuildArch = architecture
2702                         if (architecture == "arm64" && os != "Windows_NT") {
2703                             // Use the x64 test build for arm64 unix
2704                             inputWindowsTestBuildArch = "x64"
2705                         }
2706
2707                         if (isJitStressScenario(scenario)) {
2708                             inputWindowsTestsBuildName = projectFolder + '/' +
2709                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2710                         } else {
2711                             inputWindowsTestsBuildName = projectFolder + '/' +
2712                                 Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2713                         }
2714                     } // if (windowsArmJob != true)
2715
2716                     def serverGCString = ''
2717                     def testOpts = ''
2718
2719                     if (windowsArmJob != true) {
2720                         // Enable Server GC for Ubuntu PR builds
2721                         if (os == 'Ubuntu' && isPR) {
2722                             serverGCString = '--useServerGC'
2723                         }
2724
2725                         if (isR2RScenario(scenario)) {
2726
2727                             testOpts += ' --crossgen --runcrossgentests'
2728
2729                             if (scenario == 'r2r_jitstress1') {
2730                                 testOpts += ' --jitstress=1'
2731                             }
2732                             else if (scenario == 'r2r_jitstress2') {
2733                                 testOpts += ' --jitstress=2'
2734                             }
2735                             else if (scenario == 'r2r_jitstressregs1') {
2736                                 testOpts += ' --jitstressregs=1'
2737                             }
2738                             else if (scenario == 'r2r_jitstressregs2') {
2739                                 testOpts += ' --jitstressregs=2'
2740                             }
2741                             else if (scenario == 'r2r_jitstressregs3') {
2742                                 testOpts += ' --jitstressregs=3'
2743                             }
2744                             else if (scenario == 'r2r_jitstressregs4') {
2745                                 testOpts += ' --jitstressregs=4'
2746                             }
2747                             else if (scenario == 'r2r_jitstressregs8') {
2748                                 testOpts += ' --jitstressregs=8'
2749                             }
2750                             else if (scenario == 'r2r_jitstressregs0x10') {
2751                                 testOpts += ' --jitstressregs=0x10'
2752                             }
2753                             else if (scenario == 'r2r_jitstressregs0x80') {
2754                                 testOpts += ' --jitstressregs=0x80'
2755                             }
2756                             else if (scenario == 'r2r_jitstressregs0x1000') {
2757                                 testOpts += ' --jitstressregs=0x1000'
2758                             }
2759                             else if (scenario == 'r2r_jitminopts') {
2760                                 testOpts += ' --jitminopts'
2761                             }
2762                             else if (scenario == 'r2r_jitforcerelocs') {
2763                                 testOpts += ' --jitforcerelocs'
2764                             }
2765                             else if (scenario == 'r2r_gcstress15') {
2766                                 testOpts += ' --gcstresslevel=0xF'
2767                             }
2768                         }
2769                         else if (scenario == 'jitdiff') {
2770                             testOpts += ' --jitdisasm --crossgen'
2771                         }
2772                         else if (scenario == 'illink') {
2773                             testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2774                         }
2775                         else if (isLongGc(scenario)) {
2776                             // Long GC tests behave very poorly when they are not
2777                             // the only test running (many of them allocate until OOM).
2778                             testOpts += ' --sequential'
2779
2780                             // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2781                             // for running long GC and GCSimulator tests, respectively. We don't use them
2782                             // here because using a playlist file produces much more readable output on the CI machines
2783                             // and reduces running time.
2784                             //
2785                             // The Long GC playlist contains all of the tests that are
2786                             // going to be run. The GCSimulator playlist contains all of
2787                             // the GC simulator tests.
2788                             if (scenario == 'longgc') {
2789                                 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2790                             }
2791                             else if (scenario == 'gcsimulator') {
2792                                 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2793                             }
2794                         }
2795                         else if (isGcReliabilityFramework(scenario)) {
2796                             testOpts += ' --build-overlay-only'
2797                         }
2798                         else if (scenario == 'standalone_gc') {
2799                             if (osGroup == 'OSX') {
2800                                 testOpts += ' --gcname=libclrgc.dylib'
2801                             }
2802                             else if (osGroup == 'Linux') {
2803                                 testOpts += ' --gcname=libclrgc.so'
2804                             }
2805                             else {
2806                                 println("Unexpected OS group: ${osGroup} for os ${os}")
2807                                 assert false
2808                             }
2809                         }
2810                     } // if (windowsArmJob != true)
2811
2812                     def folder = getJobFolder(scenario)
2813                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2814                         // Add parameters for the inputs
2815
2816                         if (windowsArmJob == true) {
2817                             parameters {
2818                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2819                             }
2820                         }
2821                         else {
2822                             parameters {
2823                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
2824                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2825                             }
2826                         }
2827
2828                         steps {
2829                             // Set up the copies
2830
2831                             // Coreclr build containing the tests and mscorlib
2832                             // pri1 jobs still need to copy windows_nt built tests
2833                             if (windowsArmJob != true) {
2834                                 copyArtifacts(inputWindowsTestsBuildName) {
2835                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2836                                     buildSelector {
2837                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2838                                     }
2839                                 }
2840                             }
2841
2842                             // Coreclr build we are trying to test
2843                             //
2844                             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2845
2846                             copyArtifacts(inputCoreCLRBuildName) {
2847                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2848                                 buildSelector {
2849                                     buildNumber('${CORECLR_BUILD}')
2850                                 }
2851                             }
2852
2853                             // Windows CoreCLR Arm(64) will restore corefx
2854                             // packages correctly.
2855                             //
2856                             // In addition, test steps are entirely different
2857                             // because we do not have a unified runner
2858                             if (windowsArmJob != true) {
2859                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2860
2861                                 // HACK -- Arm64 does not have corefx jobs yet.
2862                                 // Clone corefx and build the native packages overwriting the x64 packages.
2863                                 if (architecture == 'arm64') {
2864                                     shell("mkdir -p ./bin/CoreFxBinDir")
2865                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2866                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2867                                 }
2868                                 else if (architecture == 'x86') {
2869                                     shell("mkdir ./bin/CoreFxNative")
2870
2871                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2872                                         includePatterns('bin/build.tar.gz')
2873                                         targetDirectory('bin/CoreFxNative')
2874                                         buildSelector {
2875                                             latestSuccessful(true)
2876                                         }
2877                                     }
2878
2879                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2880                                 }
2881
2882                                 // Unzip the tests first.  Exit with 0
2883                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2884                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2885
2886                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2887
2888                                 // Execute the tests
2889                                 def runDocker = isNeedDocker(architecture, os, false)
2890                                 def dockerPrefix = ""
2891                                 def dockerCmd = ""
2892                                 if (runDocker) {
2893                                     def dockerImage = getDockerImageName(architecture, os, false)
2894                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2895                                     dockerCmd = dockerPrefix + "${dockerImage} "
2896                                 }
2897
2898                                 // If we are running a stress mode, we'll set those variables first
2899                                 def testEnvOpt = ""
2900                                 if (isJitStressScenario(scenario)) {
2901                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2902                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2903                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2904                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2905                                     shell("${envScriptCmds}")
2906                                     testEnvOpt = "--test-env=" + scriptFileName
2907                                 }
2908
2909                                 if (isGCStressRelatedTesting(scenario)) {
2910                                     shell('./init-tools.sh')
2911                                 }
2912
2913                                 shell("""${dockerCmd}./tests/runtest.sh \\
2914                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2915                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2916                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2917                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2918
2919                                 if (isGcReliabilityFramework(scenario)) {
2920                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2921                                     if (serverGCString != '') {
2922                                         if (runDocker) {
2923                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2924                                         }
2925                                         else {
2926                                             shell("export COMPlus_gcServer=1")
2927                                         }
2928                                     }
2929
2930                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2931                                 }
2932                             }
2933                             else { // windowsArmJob == true
2934
2935                                 if (isCoreFxScenario(scenario)) {
2936
2937                                     // Only arm supported for corefx testing now.
2938                                     assert architecture == 'arm'
2939
2940                                     // Unzip CoreFx runtime
2941                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2942
2943                                     // Unzip CoreFx tests.
2944                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2945
2946                                     // Add the script to run the corefx tests
2947                                     def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2948                                     def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2949                                     def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2950                                     batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2951
2952                                 } else { // !isCoreFxScenario(scenario)
2953
2954                                     // Unzip tests.
2955                                     batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2956
2957                                     def buildCommands = ""
2958
2959                                     def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2960                                     def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2961                                     def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2962
2963                                     // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2964                                     addCommand("SETLOCAL ENABLEEXTENSIONS")
2965
2966                                     // For all jobs
2967                                     addEnvVariable("CORE_ROOT", coreRootLocation)
2968
2969                                     addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2970                                     addEnvVariable("COMPlus_ContinueOnAssert", "0")
2971
2972                                     // ARM legacy backend; this is an altjit.
2973                                     if (architecture == "armlb") {
2974                                         addEnvVariable("COMPlus_AltJit", "*")
2975                                         addEnvVariable("COMPlus_AltJitNgen", "*")
2976                                         addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2977                                         addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2978                                     }
2979
2980                                     // If we are running a stress mode, we'll set those variables as well
2981                                     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2982                                         def stressValues = null
2983                                         if (isJitStressScenario(scenario)) {
2984                                             stressValues = Constants.jitStressModeScenarios[scenario]
2985                                         }
2986                                         else {
2987                                             stressValues = Constants.r2rStressScenarios[scenario]
2988                                         }
2989
2990                                         stressValues.each { key, value ->
2991                                             addEnvVariable(key, value)
2992                                         }
2993                                     }
2994
2995                                     if (isR2RScenario(scenario)) {
2996                                         // Crossgen the framework assemblies.
2997                                         buildCommands += """
2998 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2999 @goto skip_PrecompileAssembly
3000
3001 :PrecompileAssembly
3002 @REM Skip mscorlib since it is already precompiled.
3003 @if /I "%3" == "mscorlib.dll" exit /b 0
3004 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3005
3006 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3007 @if "%errorlevel%" == "-2146230517" (
3008     echo %2 is not a managed assembly.
3009 ) else if "%errorlevel%" == "-2146234344" (
3010     echo %2 is not a managed assembly.
3011 ) else if %errorlevel% neq 0 (
3012     echo Unable to precompile %2
3013 ) else (
3014     echo Precompiled %2
3015 )
3016 @exit /b 0
3017
3018 :skip_PrecompileAssembly
3019 """
3020
3021                                         // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3022                                         // crossgen on tests before running them.
3023                                         addEnvVariable("RunCrossGen", "true")
3024                                     } // isR2RScenario(scenario)
3025
3026                                     // Create the smarty command
3027                                     def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3028                                     def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3029                                     def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3030
3031                                     def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
3032
3033                                     if (architecture == "armlb") {
3034                                         addExclude("LEGACYJIT_FAIL")
3035                                     }
3036
3037                                     // Exclude tests based on scenario.
3038                                     Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3039                                         addArchSpecificExclude(architecture, excludeTag)
3040                                     }
3041
3042                                     // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
3043                                     if (scenario == 'innerloop') {
3044                                         addExclude("pri1")
3045                                     }
3046
3047                                     // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3048                                     // TODO: We should create a "long running" job that runs these with a longer timeout.
3049                                     addExclude("LONG_RUNNING")
3050
3051                                     smartyCommand += "/lstFile Tests.lst"
3052
3053                                     def testListArch = [
3054                                         'arm64': 'arm64',
3055                                         'arm': 'arm',
3056                                         'armlb': 'arm'
3057                                     ]
3058
3059                                     def archLocation = testListArch[architecture]
3060
3061                                     addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3062                                     addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3063                                     addCommand("${smartyCommand}")
3064
3065                                     // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3066                                     // However, we also need to remove all the variables that were set during this batch file, so we
3067                                     // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3068                                     // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3069                                     // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3070                                     // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3071                                     // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3072                                     addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3073
3074                                     // ZIP up the smarty output, no matter what the smarty result.
3075                                     addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3076
3077                                     addCommand("echo %errorlevel%")
3078                                     addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3079
3080                                     // Use the smarty errorlevel as the script errorlevel.
3081                                     addCommand("exit /b %__save_smarty_errorlevel%")
3082
3083                                     batchFile(buildCommands)
3084                                 } // non-corefx testing
3085                             } // windowsArmJob == true
3086                         } // steps
3087                     } // job
3088
3089                     addToViews(newJob, isPR, architecture, os)
3090
3091                     if (scenario == 'jitdiff') {
3092                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3093                     }
3094
3095                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3096                     if (os in ['Ubuntu']) {
3097                         SummaryBuilder summaries = new SummaryBuilder()
3098                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3099                         summaries.emit(newJob)
3100                     }
3101
3102                     def affinityOptions = null
3103
3104                     if (windowsArmJob == true) {
3105                         affinityOptions = [
3106                             "use_arm64_build_machine" : false
3107                         ]
3108                     }
3109
3110                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3111                         affinityOptions = [
3112                             "large_pages" : false
3113                         ]
3114                     }
3115
3116                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3117                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3118
3119                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3120
3121                     if (windowsArmJob != true) {
3122                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3123                     }
3124                     else {
3125                         if (!isCoreFxScenario(scenario)) {
3126                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3127
3128                             // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3129                             // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3130                             // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3131                             Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3132                         }
3133                     }
3134
3135                     // =============================================================================================
3136                     // Create a build flow to join together the build and tests required to run this test.
3137                     // =============================================================================================
3138
3139                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3140                     // Linux CoreCLR test
3141                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3142                     def fullTestJobName = projectFolder + '/' + newJob.name
3143                     // Add a reference to the input jobs for report purposes
3144                     JobReport.Report.addReference(inputCoreCLRBuildName)
3145                     if (windowsArmJob != true) {
3146                         JobReport.Report.addReference(inputWindowsTestsBuildName)
3147                     }
3148                     JobReport.Report.addReference(fullTestJobName)
3149                     def newFlowJob = null
3150
3151                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3152                         // Do not create the flow job for RHEL jobs.
3153                         return
3154                     }
3155
3156                     // For pri0 jobs we can build tests on unix
3157                     if (windowsArmJob) {
3158                         // For Windows arm jobs there is no reason to build a parallel test job.
3159                         // The product build supports building and archiving the tests.
3160
3161                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3162                         buildFlow("""\
3163 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3164
3165 // And then build the test build
3166 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3167 """)
3168                         }
3169                     }
3170                     else {
3171                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3172                         buildFlow("""\
3173 // Build the input jobs in parallel
3174 parallel (
3175 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3176 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3177 )
3178
3179 // And then build the test build
3180 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3181             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3182 """)
3183                         }
3184                     }
3185
3186                     addToViews(newFlowJob, isPR, architecture, os)
3187
3188                     // For the flow jobs set the machine affinity as x64 if an armarch.
3189                     def flowArch = architecture
3190
3191                     if (flowArch in validWindowsNTCrossArches) {
3192                         flowArch = 'x64'
3193                         affinityOptions = null
3194                     }
3195
3196                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3197                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3198                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3199                 } // configuration
3200             } // os
3201         } // architecture
3202     } // isPR
3203 } // scenario
3204
3205 JobReport.Report.generateJobReport(out)
3206
3207 // Make the call to generate the help job
3208 Utilities.createHelperJob(this, project, branch,
3209     "Welcome to the ${project} Repository",  // This is prepended to the help message
3210     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3211
3212 Utilities.addCROSSCheck(this, project, branch)