netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213         'Tizen': [
 214             'armem': [
 215                 'Checked'
 216             ]
 217         ],
 218     ]
 219
 220     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 221     // to Tests.lst file categories to exclude.
 222     //
 223     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 224     // and with the same values, with some commented out, for easier maintenance.
 225     //
 226     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 227     //
 228     def static validArmWindowsScenarios = [
 229                'innerloop':                              [],
 230                'normal':                                 [],
 231                // 'ilrt'
 232                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 233                // 'longgc'
 234                // 'formatting'
 235                // 'gcsimulator'
 236                // 'jitdiff'
 237                // 'standalone_gc'
 238                // 'gc_reliability_framework'
 239                // 'illink'
 240                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 251                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 253                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 254                'tieredcompilation':                      [],
 255                'forcerelocs':                            [],
 256                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 275                // 'jitsse2only'                          // Only relevant to xarch
 276                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 277                // 'jitincompletehwintrinsic'
 278                // 'jitx86hwintrinsicnoavx'
 279                // 'jitx86hwintrinsicnoavx2'
 280                // 'jitx86hwintrinsicnosimd'
 281                // 'jitnox86hwintrinsic'
 282                'corefx_baseline':                        [],    // corefx tests don't use smarty
 283                'corefx_minopts':                         [],    // corefx tests don't use smarty
 284                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 285                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 286                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 290                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 291                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 292                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 293                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 294                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 295                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 296                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 297                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 298                'heapverify1':                            [],
 299                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 300                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 302                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 305
 306                //
 307                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 308                //
 309
 310                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 311                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 324                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 325                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 326                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 327                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 328                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 329     ]
 330
 331     def static validLinuxArm64Scenarios = [
 332                'innerloop',
 333                'normal',
 334                'r2r',
 335                'gcstress0x3',
 336                'gcstress0xc'
 337     ]
 338
 339     // Note: no GCStress-related scenario is enabled currently.
 340     def static validLinuxArmScenarios = [
 341                'innerloop',
 342                'normal',
 343                'r2r',
 344                'r2r_jitstress1',
 345                'r2r_jitstress2',
 346                'r2r_jitstressregs1',
 347                'r2r_jitstressregs2',
 348                'r2r_jitstressregs3',
 349                'r2r_jitstressregs4',
 350                'r2r_jitstressregs8',
 351                'r2r_jitstressregs0x10',
 352                'r2r_jitstressregs0x80',
 353                'r2r_jitstressregs0x1000',
 354                'r2r_jitminopts',
 355                'r2r_jitforcerelocs',
 356                // 'r2r_gcstress15',
 357                'minopts',
 358                'forcerelocs',
 359                'jitstress1',
 360                'jitstress2',
 361                'jitstressregs1',
 362                'jitstressregs2',
 363                'jitstressregs3',
 364                'jitstressregs4',
 365                'jitstressregs8',
 366                'jitstressregs0x10',
 367                'jitstressregs0x80',
 368                'jitstressregs0x1000',
 369                'jitstress2_jitstressregs1',
 370                'jitstress2_jitstressregs2',
 371                'jitstress2_jitstressregs3',
 372                'jitstress2_jitstressregs4',
 373                'jitstress2_jitstressregs8',
 374                'jitstress2_jitstressregs0x10',
 375                'jitstress2_jitstressregs0x80',
 376                'jitstress2_jitstressregs0x1000',
 377                'tailcallstress'
 378                // 'gcstress0x3',
 379                // 'gcstress0xc',
 380                // 'zapdisable',
 381                // 'heapverify1',
 382                // 'gcstress0xc_zapdisable',
 383                // 'gcstress0xc_zapdisable_jitstress2',
 384                // 'gcstress0xc_zapdisable_heapverify1',
 385                // 'gcstress0xc_jitstress1',
 386                // 'gcstress0xc_jitstress2',
 387                // 'gcstress0xc_minopts_heapverify1'
 388     ]
 389
 390     def static configurationList = ['Debug', 'Checked', 'Release']
 391
 392     // This is the set of architectures
 393     // Some of these are pseudo-architectures:
 394     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 395     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 396     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 397     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 398     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 399
 400     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 401     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 402 }
 403
 404 // **************************************************************
 405 // Create some specific views
 406 //
 407 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 408 // views based on a single regular expression. These views will be generated by adding a
 409 // specific set of jobs to them.
 410 //
 411 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 412 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 413 // those are required, don't add them (which simplifies the view pages, as well).
 414 // **************************************************************
 415
 416 class Views {
 417     def static MergeJobView = null
 418     def static PeriodicJobView = null
 419     def static ArchitectureViews = [:]
 420     def static OSViews = [:]
 421 }
 422
 423 // MergeJobView: include all jobs that execute when a PR change is merged.
 424 Views.MergeJobView = listView('Merge') {
 425     recurse()
 426     columns {
 427         status()
 428         weather()
 429         name()
 430         lastSuccess()
 431         lastFailure()
 432         lastDuration()
 433         buildButton()
 434     }
 435 }
 436
 437 // PeriodicJobView: include all jobs that execute on a schedule
 438 Views.PeriodicJobView = listView('Periodic') {
 439     recurse()
 440     columns {
 441         status()
 442         weather()
 443         name()
 444         lastSuccess()
 445         lastFailure()
 446         lastDuration()
 447         buildButton()
 448     }
 449 }
 450
 451 // Create a view for non-PR jobs for each architecture.
 452 Constants.architectureList.each { architecture ->
 453     Views.ArchitectureViews[architecture] = listView(architecture) {
 454         recurse()
 455         columns {
 456             status()
 457             weather()
 458             name()
 459             lastSuccess()
 460             lastFailure()
 461             lastDuration()
 462             buildButton()
 463         }
 464     }
 465 }
 466
 467 // Create a view for non-PR jobs for each OS.
 468 Constants.osList.each { os ->
 469     // Don't create one for the special 'Windows_NT_BuildOnly'
 470     if (os == 'Windows_NT_BuildOnly') {
 471         return
 472     }
 473     Views.OSViews[os] = listView(os) {
 474         recurse()
 475         columns {
 476             status()
 477             weather()
 478             name()
 479             lastSuccess()
 480             lastFailure()
 481             lastDuration()
 482             buildButton()
 483         }
 484     }
 485 }
 486
 487 def static addToMergeView(def job) {
 488     Views.MergeJobView.with {
 489         jobs {
 490             name(job.name)
 491         }
 492     }
 493 }
 494
 495 def static addToPeriodicView(def job) {
 496     Views.PeriodicJobView.with {
 497         jobs {
 498             name(job.name)
 499         }
 500     }
 501 }
 502
 503 def static addToViews(def job, def isPR, def architecture, def os) {
 504     if (isPR) {
 505         // No views want PR jobs currently.
 506         return
 507     }
 508
 509     // Add to architecture view.
 510     Views.ArchitectureViews[architecture].with {
 511         jobs {
 512             name(job.name)
 513         }
 514     }
 515
 516     // Add to OS view.
 517     Views.OSViews[os].with {
 518         jobs {
 519             name(job.name)
 520         }
 521     }
 522 }
 523
 524 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 525     addToPeriodicView(job)
 526     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 527 }
 528
 529 def static addGithubPushTriggerHelper(def job) {
 530     addToMergeView(job)
 531     Utilities.addGithubPushTrigger(job)
 532 }
 533
 534
 535 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 536     assert os instanceof String
 537     assert architecture instanceof String
 538
 539     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 540     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 541
 542     if (!(architecture in armArches)) {
 543         assert options == null
 544         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 545
 546         return
 547     }
 548
 549     // This is an arm(64) job.
 550     //
 551     // There are several options.
 552     //
 553     // Windows_NT
 554     //
 555     // Arm32 (Build) -> latest-arm64
 556     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 557     // Arm32 (Test)  -> arm64-windows_nt
 558     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 559     //
 560     // Arm64 (Build) -> latest-arm64
 561     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 562     // Arm64 (Test)  -> arm64-windows_nt
 563     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 564     //
 565     // Ubuntu
 566     //
 567     // Arm32 emulator (Build, Test) -> arm-cross-latest
 568     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 569     //
 570     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 571     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 572     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 573     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 574     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 575     //       |-> os == "Ubuntu" && (architecture == "arm")
 576     //
 577     // Arm64 (Build) -> arm64-cross-latest
 578     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 579     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 580     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 581     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 582     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 583
 584     // This has to be a arm arch
 585     assert architecture in armArches
 586     if (os == "Windows_NT") {
 587         // arm32/arm64 Windows jobs share the same machines for now
 588         def isBuild = options['use_arm64_build_machine'] == true
 589
 590         if (isBuild == true) {
 591             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 592         } else {
 593             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 594         }
 595     } else {
 596         assert os != 'Windows_NT'
 597         assert os in supportedArmLinuxOs
 598
 599         if (architecture == 'arm64') {
 600             if ((options != null) && (options['is_build_only'] == true)) {
 601                 // Arm64 Linux build machine
 602                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 603             } else {
 604                 // Arm64 Linux test machines
 605                 if ((options != null) && (options['large_pages'] == true)) {
 606                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 607                 } else {
 608                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 609                 }
 610             }
 611         }
 612         else if (architecture == 'armem') {
 613             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 614             // using Docker.
 615             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 616         }
 617         else {
 618             // arm Ubuntu on hardware.
 619             assert (architecture == 'arm') && (os == 'Ubuntu')
 620             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 621             def isBuild = (options != null) && (options['is_build_job'] == true)
 622             if (isFlow || isBuild) {
 623                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 624                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 625                 // arm hardware.
 626                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 627             } else {
 628                 // arm Ubuntu test machine
 629                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 630                 // Utilities.setMachineAffinity. Just add the machine affinity
 631                 // manually. We specify the Helix queue name here.
 632                 job.with {
 633                     label('ubuntu.1404.arm32.open')
 634                 }
 635             }
 636         }
 637     }
 638 }
 639
 640 // setJobMachineAffinity: compute the machine affinity options for a job,
 641 // then set the job with those affinity options.
 642 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 643 {
 644     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 645            (!isBuildJob && isTestJob  && !isFlowJob) ||
 646            (!isBuildJob && !isTestJob && isFlowJob)
 647
 648     def affinityOptions = null
 649     def affinityArchitecture = architecture
 650
 651     if (os == "Windows_NT") {
 652         if (architecture in Constants.armWindowsCrossArchitectureList) {
 653             if (isBuildJob) {
 654                 affinityOptions = [ "use_arm64_build_machine" : true ]
 655             } else if (isTestJob) {
 656                 affinityOptions = [ "use_arm64_build_machine" : false ]
 657             } else if (isFlowJob) {
 658                 // For the flow jobs set the machine affinity as x64
 659                 affinityArchitecture = 'x64'
 660             }
 661         }
 662     }
 663     else {
 664         if (architecture == 'arm64') {
 665             if (isBuildJob) {
 666                 affinityOptions = ['is_build_only': true]
 667             } else if (isTestJob) {
 668                 affinityOptions = [ "large_pages" : false ]
 669             }
 670         }
 671         else if (architecture == 'arm') {
 672             if (isBuildJob) {
 673                 affinityOptions = ['is_build_job': true]
 674             } else if (isFlowJob) {
 675                 affinityOptions = ['is_flow_job': true]
 676             }
 677         }
 678     }
 679
 680     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 681 }
 682
 683 def static isGCStressRelatedTesting(def scenario) {
 684     // The 'r2r_gcstress15' scenario is a basic scenario.
 685     // Detect it and make it a GCStress related.
 686     if (scenario == 'r2r_gcstress15')
 687     {
 688         return true;
 689     }
 690
 691     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 692     def scenarioName = scenario.toLowerCase()
 693     def isGCStressTesting = false
 694     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 695         if (k in gcStressTestEnvVars) {
 696             isGCStressTesting = true;
 697         }
 698     }
 699     return isGCStressTesting
 700 }
 701
 702 def static isCoreFxScenario(def scenario) {
 703     def corefx_prefix = 'corefx_'
 704     if (scenario.length() < corefx_prefix.length()) {
 705         return false
 706     }
 707     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 708 }
 709
 710 def static isR2RBaselineScenario(def scenario) {
 711     return (scenario == 'r2r')
 712 }
 713
 714 def static isR2RStressScenario(def scenario) {
 715     return Constants.r2rStressScenarios.containsKey(scenario)
 716 }
 717
 718 def static isR2RScenario(def scenario) {
 719     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 720 }
 721
 722 def static isJitStressScenario(def scenario) {
 723     return Constants.jitStressModeScenarios.containsKey(scenario)
 724 }
 725
 726 def static isLongGc(def scenario) {
 727     return (scenario == 'longgc' || scenario == 'gcsimulator')
 728 }
 729
 730 def static isJitDiff(def scenario) {
 731     return (scenario == 'jitdiff')
 732 }
 733
 734 def static isGcReliabilityFramework(def scenario) {
 735     return (scenario == 'gc_reliability_framework')
 736 }
 737
 738 def static isArmWindowsScenario(def scenario) {
 739     return Constants.validArmWindowsScenarios.containsKey(scenario)
 740 }
 741
 742 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 743     if (isBuildOnly == true) {
 744         os = 'Windows_NT_BuildOnly'
 745     }
 746
 747     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 748
 749     if (validOsPrTriggerArchConfigs == null) {
 750         return false
 751     }
 752
 753     if (validOsPrTriggerArchConfigs[architecture] != null) {
 754         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 755
 756         if (!(configuration in validOsPrTriggerConfigs)) {
 757             return false
 758         }
 759     } else {
 760         return false
 761     }
 762
 763     return true
 764 }
 765
 766 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 767     // 2 hours (120 minutes) is the default timeout
 768     def timeout = 120
 769     def innerLoop = (scenario == "innerloop")
 770
 771     if (!innerLoop) {
 772         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 773         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 774         timeout = 240
 775     }
 776
 777     if (!isBuildOnly) {
 778         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 779         if (isGCStressRelatedTesting(scenario)) {
 780             timeout = 4320
 781         }
 782         else if (isCoreFxScenario(scenario)) {
 783             timeout = 360
 784         }
 785         else if (isJitStressScenario(scenario)) {
 786             timeout = 300
 787         }
 788         else if (isR2RBaselineScenario(scenario)) {
 789             timeout = 240
 790         }
 791         else if (isLongGc(scenario)) {
 792             timeout = 1440
 793         }
 794         else if (isJitDiff(scenario)) {
 795             timeout = 240
 796         }
 797         else if (isGcReliabilityFramework(scenario)) {
 798             timeout = 1440
 799         }
 800         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 801             timeout = 240
 802         }
 803
 804         if (architecture == 'arm') {
 805             // ARM32 machines are particularly slow.
 806             timeout += 120
 807         }
 808     }
 809
 810     if (configuration == 'Debug') {
 811         // Debug runs can be very slow. Add an hour.
 812         timeout += 60
 813     }
 814
 815     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 816         // AltJit runs compile all methods twice.
 817         timeout *= 2
 818     }
 819
 820     // If we've changed the timeout from the default, set it in the job.
 821
 822     if (timeout != 120) {
 823         Utilities.setJobTimeout(newJob, timeout)
 824     }
 825 }
 826
 827 def static getJobFolder(def scenario) {
 828     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 829         return 'jitstress'
 830     }
 831     if (scenario == 'illink') {
 832         return 'illink'
 833     }
 834     return ''
 835 }
 836
 837 def static getStressModeDisplayName(def scenario) {
 838     def displayStr = ''
 839     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 840         def prefixLength = 'COMPlus_'.length()
 841         if (k.length() >= prefixLength) {
 842             def modeName = k.substring(prefixLength, k.length())
 843             displayStr += ' ' + modeName + '=' + v
 844         }
 845     }
 846
 847     if (isCoreFxScenario(scenario)) {
 848         displayStr = ('CoreFx ' + displayStr).trim()
 849     }
 850
 851     return displayStr
 852 }
 853
 854 def static getR2RDisplayName(def scenario) {
 855     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 856     def displayStr = scenario
 857     def prefixLength = 'r2r_'.length()
 858     if (displayStr.length() >= prefixLength) {
 859         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 860     } else if (scenario == 'r2r') {
 861         displayStr = "R2R"
 862     }
 863     return displayStr
 864 }
 865
 866 //
 867 // Functions to create an environment script.
 868 //      envScriptCreate -- initialize the script (call first)
 869 //      envScriptFinalize -- finalize the script (call last)
 870 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 871 //      envScriptAppendExistingScript -- append an existing script to the generated script
 872 //
 873 // Each script returns a string of commands. Concatenate all the strings together before
 874 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 875 //
 876
 877 // Initialize the environment setting script.
 878 def static envScriptCreate(def os, def stepScriptLocation) {
 879     def stepScript = ''
 880     if (os == 'Windows_NT') {
 881         stepScript += "echo Creating TestEnv script\r\n"
 882         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 883
 884         // Create at least an empty script.
 885         stepScript += "echo. > ${stepScriptLocation}\r\n"
 886     }
 887     else {
 888         stepScript += "echo Creating environment setting script\n"
 889         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 890     }
 891
 892     return stepScript
 893 }
 894
 895 // Generates the string for setting stress mode variables.
 896 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 897     def stepScript = ''
 898     if (os == 'Windows_NT') {
 899         stressModeVars.each{ k, v ->
 900             // Write out what we are writing to the script file
 901             stepScript += "echo Setting ${k}=${v}\r\n"
 902             // Write out the set itself to the script file`
 903             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 904         }
 905     }
 906     else {
 907         stressModeVars.each{ k, v ->
 908             // Write out what we are writing to the script file
 909             stepScript += "echo Setting ${k}=${v}\n"
 910             // Write out the set itself to the script file`
 911             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 912         }
 913     }
 914
 915     return stepScript
 916 }
 917
 918 // Append an existing script to an environment script.
 919 // Returns string of commands to do this.
 920 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 921     assert (os == 'Windows_NT')
 922     def stepScript = ''
 923
 924     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 925     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 926
 927     return stepScript
 928 }
 929
 930 // Finalize an environment setting script.
 931 // Returns string of commands to do this.
 932 def static envScriptFinalize(def os, def stepScriptLocation) {
 933     def stepScript = ''
 934
 935     if (os == 'Windows_NT') {
 936         // Display the resulting script. This is useful when looking at the output log file.
 937         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 938         stepScript += "type ${stepScriptLocation}\r\n"
 939     }
 940     else {
 941         stepScript += "chmod +x ${stepScriptLocation}\n"
 942     }
 943
 944     return stepScript
 945 }
 946
 947 def static isNeedDocker(def architecture, def os, def isBuild) {
 948     if (isBuild) {
 949         if (architecture == 'x86' && os == 'Ubuntu') {
 950             return true
 951         }
 952         else if (architecture == 'armem') {
 953             return true
 954         }
 955         else if (architecture == 'arm') {
 956             if (os == 'Ubuntu') {
 957                 return true
 958             }
 959         }
 960     }
 961     else {
 962         if (architecture == 'x86' && os == 'Ubuntu') {
 963             return true
 964         }
 965     }
 966     return false
 967 }
 968
 969 def static getDockerImageName(def architecture, def os, def isBuild) {
 970     // We must change some docker private images to official later
 971     if (isBuild) {
 972         if (architecture == 'x86' && os == 'Ubuntu') {
 973             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 974         }
 975         else if (architecture == 'armem') {
 976             if (os == 'Ubuntu') {
 977                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 978             }
 979             else if (os == 'Ubuntu16.04') {
 980                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 981             }
 982             else if (os == 'Tizen') {
 983                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 984             }
 985         }
 986         else if (architecture == 'arm') {
 987             if (os == 'Ubuntu') {
 988                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20170319080304"
 989             }
 990         }
 991     }
 992     else {
 993         if (architecture == 'x86' && os == 'Ubuntu') {
 994             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 995         }
 996     }
 997     println("Unknown architecture to use docker: ${architecture} ${os}");
 998     assert false
 999 }
1000
1001
1002 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
1003 def static jobRequiresLimitedHardware(def architecture, def os) {
1004     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
1005         // These test jobs require ARM64 hardware
1006         return true
1007     }
1008     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1009         // These test jobs require Linux/arm32 hardware
1010         return true
1011     }
1012     else {
1013         return false
1014     }
1015 }
1016
1017 // Calculates the name of the build job based on some typical parameters.
1018 //
1019 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1020     // If the architecture is x64, do not add that info into the build name.
1021     // Need to change around some systems and other builds to pick up the right builds
1022     // to do that.
1023
1024     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1025     if (isBuildOnly) {
1026         suffix += '_bld'
1027     }
1028     def baseName = ''
1029     switch (architecture) {
1030         case 'x64':
1031             if (scenario == 'normal') {
1032                 // For now we leave x64 off of the name for compatibility with other jobs
1033                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1034             }
1035             else if (scenario == 'formatting') {
1036                 // we don't care about the configuration for the formatting job. It runs all configs
1037                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1038             }
1039             else {
1040                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1041             }
1042             break
1043         case 'arm64':
1044             if (os.toLowerCase() == "windows_nt") {
1045                 // These are cross builds
1046                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1047             }
1048             else {
1049                 // Defaults to a small page size set of machines.
1050                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1051             }
1052             break
1053         case 'armem':
1054             // These are cross builds
1055             if (os == 'Tizen') {
1056                 // ABI: softfp
1057                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1058             }
1059             else {
1060                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1061             }
1062             break
1063         case 'armlb':
1064         case 'arm':
1065             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1066             break
1067         case 'x86':
1068         case 'x86_arm_altjit':
1069         case 'x64_arm64_altjit':
1070             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1071             break
1072         default:
1073             println("Unknown architecture: ${architecture}");
1074             assert false
1075             break
1076     }
1077
1078     return baseName + suffix
1079 }
1080
1081 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1082
1083     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1084     if (os == 'Windows_NT') {
1085         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1086             if (!(branch in Constants.WindowsArm64Branches)) {
1087                 return
1088             }
1089         }
1090     }
1091
1092     // Check scenario.
1093     switch (scenario) {
1094         case 'innerloop':
1095             break
1096         case 'normal':
1097             switch (architecture) {
1098                 case 'x64':
1099                 case 'x86':
1100                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1101                         addPeriodicTriggerHelper(job, '@daily')
1102                     }
1103                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     break
1107                 case 'arm':
1108                     if (os == 'Windows_NT') {
1109                         addGithubPushTriggerHelper(job)
1110                     }
1111                     else {
1112                         // Currently no push triggers, with limited arm Linux hardware.
1113                         assert os == 'Ubuntu'
1114                         addPeriodicTriggerHelper(job, '@daily')
1115                     }
1116                     break
1117                 case 'armem':
1118                 case 'armlb':
1119                 case 'x86_arm_altjit':
1120                 case 'x64_arm64_altjit':
1121                     addGithubPushTriggerHelper(job)
1122                     break
1123                 case 'arm64':
1124                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1125                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1126                     break
1127                 default:
1128                     println("Unknown architecture: ${architecture}");
1129                     assert false
1130                     break
1131             }
1132             break
1133         case 'r2r':
1134             assert !(os in bidailyCrossList)
1135             // r2r gets a push trigger for checked/release
1136             if (configuration == 'Checked' || configuration == 'Release') {
1137                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1138                 if (architecture == 'x64' && os != 'OSX10.12') {
1139                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1140                     if (isFlowJob || os == 'Windows_NT') {
1141                         addGithubPushTriggerHelper(job)
1142                     }
1143                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1144                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1145                     if (isFlowJob) {
1146                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1147                     }
1148                 }
1149                 // For x86, only add per-commit jobs for Windows
1150                 else if (architecture == 'x86') {
1151                     if (os == 'Windows_NT') {
1152                         addGithubPushTriggerHelper(job)
1153                     }
1154                 }
1155                 // arm64 r2r jobs should only run daily.
1156                 else if (architecture == 'arm64') {
1157                     if (os == 'Windows_NT') {
1158                         addPeriodicTriggerHelper(job, '@daily')
1159                     }
1160                 }
1161             }
1162             break
1163         case 'r2r_jitstress1':
1164         case 'r2r_jitstress2':
1165         case 'r2r_jitstressregs1':
1166         case 'r2r_jitstressregs2':
1167         case 'r2r_jitstressregs3':
1168         case 'r2r_jitstressregs4':
1169         case 'r2r_jitstressregs8':
1170         case 'r2r_jitstressregs0x10':
1171         case 'r2r_jitstressregs0x80':
1172         case 'r2r_jitstressregs0x1000':
1173         case 'r2r_jitminopts':
1174         case 'r2r_jitforcerelocs':
1175         case 'r2r_gcstress15':
1176             assert !(os in bidailyCrossList)
1177
1178             // GCStress=C is currently not supported on OS X
1179             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1180                 break
1181             }
1182
1183             // GC Stress 15 r2r gets a push trigger for checked/release
1184             if (configuration == 'Checked' || configuration == 'Release') {
1185                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1186                 if (architecture == 'x64') {
1187                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1188                     if (isFlowJob || os == 'Windows_NT') {
1189                         // Add a weekly periodic trigger
1190                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1191                     }
1192                 }
1193                 // For x86, only add per-commit jobs for Windows
1194                 else if (architecture == 'x86') {
1195                     if (os == 'Windows_NT') {
1196                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1197                     }
1198                 }
1199             }
1200             break
1201         case 'longgc':
1202             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1203             assert configuration == 'Release'
1204             assert architecture == 'x64'
1205             addPeriodicTriggerHelper(job, '@daily')
1206             // TODO: Add once external email sending is available again
1207             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1208             break
1209         case 'gcsimulator':
1210             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1211             assert configuration == 'Release'
1212             assert architecture == 'x64'
1213             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1214             // TODO: Add once external email sending is available again
1215             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1216             break
1217         case 'standalone_gc':
1218             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1219             assert (configuration == 'Release' || configuration == 'Checked')
1220             // TODO: Add once external email sending is available again
1221             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1222             addPeriodicTriggerHelper(job, '@daily')
1223             break
1224         case 'gc_reliability_framework':
1225             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1226             assert (configuration == 'Release' || configuration == 'Checked')
1227             // Only triggered by phrase.
1228             break
1229         case 'ilrt':
1230             assert !(os in bidailyCrossList)
1231             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1232             if (architecture == 'x64' && configuration == 'Release') {
1233                 // We don't expect to see a job generated except in these scenarios
1234                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1235                 if (isFlowJob || os == 'Windows_NT') {
1236                     addPeriodicTriggerHelper(job, '@daily')
1237                 }
1238             }
1239             break
1240         case 'jitdiff':
1241             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1242             assert configuration == 'Checked'
1243             assert (architecture == 'x64' || architecture == 'x86')
1244             addGithubPushTriggerHelper(job)
1245             break
1246         case 'formatting':
1247             assert (os == 'Windows_NT' || os == "Ubuntu")
1248             assert architecture == 'x64'
1249             addGithubPushTriggerHelper(job)
1250             break
1251         case 'jitstressregs1':
1252         case 'jitstressregs2':
1253         case 'jitstressregs3':
1254         case 'jitstressregs4':
1255         case 'jitstressregs8':
1256         case 'jitstressregs0x10':
1257         case 'jitstressregs0x80':
1258         case 'jitstressregs0x1000':
1259         case 'minopts':
1260         case 'forcerelocs':
1261         case 'jitstress1':
1262         case 'jitstress2':
1263         case 'jitstress2_jitstressregs1':
1264         case 'jitstress2_jitstressregs2':
1265         case 'jitstress2_jitstressregs3':
1266         case 'jitstress2_jitstressregs4':
1267         case 'jitstress2_jitstressregs8':
1268         case 'jitstress2_jitstressregs0x10':
1269         case 'jitstress2_jitstressregs0x80':
1270         case 'jitstress2_jitstressregs0x1000':
1271         case 'tailcallstress':
1272         case 'jitsse2only':
1273         case 'jitnosimd':
1274         case 'jitnox86hwintrinsic':
1275         case 'jitincompletehwintrinsic':
1276         case 'jitx86hwintrinsicnoavx':
1277         case 'jitx86hwintrinsicnoavx2':
1278         case 'jitx86hwintrinsicnosimd':
1279         case 'corefx_baseline':
1280         case 'corefx_minopts':
1281         case 'corefx_jitstress1':
1282         case 'corefx_jitstress2':
1283         case 'corefx_jitstressregs1':
1284         case 'corefx_jitstressregs2':
1285         case 'corefx_jitstressregs3':
1286         case 'corefx_jitstressregs4':
1287         case 'corefx_jitstressregs8':
1288         case 'corefx_jitstressregs0x10':
1289         case 'corefx_jitstressregs0x80':
1290         case 'corefx_jitstressregs0x1000':
1291         case 'zapdisable':
1292             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1293                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1294                 if (jobRequiresLimitedHardware(architecture, os)) {
1295                     addPeriodicTriggerHelper(job, '@weekly')
1296                 }
1297                 else {
1298                     addPeriodicTriggerHelper(job, '@daily')
1299                 }
1300             }
1301             break
1302         case 'heapverify1':
1303         case 'gcstress0x3':
1304             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1305                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1306                 addPeriodicTriggerHelper(job, '@weekly')
1307             }
1308             break
1309         case 'gcstress0xc':
1310         case 'gcstress0xc_zapdisable':
1311         case 'gcstress0xc_zapdisable_jitstress2':
1312         case 'gcstress0xc_zapdisable_heapverify1':
1313         case 'gcstress0xc_jitstress1':
1314         case 'gcstress0xc_jitstress2':
1315         case 'gcstress0xc_minopts_heapverify1':
1316             // GCStress=C is currently not supported on OS X
1317             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1318                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1319                 addPeriodicTriggerHelper(job, '@weekly')
1320             }
1321             break
1322
1323         case 'illink':
1324             // Testing on other operating systems TBD
1325             assert (os == 'Windows_NT' || os == 'Ubuntu')
1326             if (architecture == 'x64' || architecture == 'x86') {
1327                 if (configuration == 'Checked') {
1328                     addPeriodicTriggerHelper(job, '@daily')
1329                 }
1330             }
1331             break
1332
1333         case 'tieredcompilation':
1334         case 'corefx_tieredcompilation':
1335             // No periodic jobs just yet, still testing
1336             break
1337
1338         default:
1339             println("Unknown scenario: ${scenario}");
1340             assert false
1341             break
1342     }
1343     return
1344 }
1345
1346 // **************************
1347 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1348 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1349 // builds will do a couple extra steps.
1350 // **************************
1351
1352 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1353 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1354 // it's just used for internal builds
1355 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1356 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1357     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1358
1359     if (isWindowsBuildOnlyJob) {
1360         return
1361     }
1362
1363     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1364     // Non pull request builds.
1365     if (!isPR) {
1366         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1367         return
1368     }
1369
1370      def arm64Users = [
1371         'AndyAyersMS',
1372         'briansull',
1373         'BruceForstall',
1374         'CarolEidt',
1375         'cmckinsey',
1376         'erozenfeld',
1377         'janvorli',
1378         'jashook',
1379         'JosephTremoulet',
1380         'pgodeq',
1381         'pgavlin',
1382         'rartemev',
1383         'russellhadley',
1384         'RussKeldorph',
1385         'sandreenko',
1386         'sdmaclea',
1387         'swaroop-sridhar',
1388         'jkotas',
1389         'markwilkie',
1390         'weshaggard'
1391     ]
1392
1393     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1394     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1395     def osGroup = getOSGroup(os)
1396     switch (architecture) {
1397         case 'x64': // editor brace matching: {
1398             if (scenario == 'formatting') {
1399                 assert configuration == 'Checked'
1400                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1401                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1402                 }
1403
1404                 break
1405             }
1406
1407             switch (os) {
1408                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1409                 case 'Debian8.4':
1410                 case 'RHEL7.2':
1411                     if (scenario == 'innerloop') {
1412                         assert !isFlowJob
1413                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1414                     }
1415                     else if (scenario == 'normal') {
1416                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1417                     }
1418                     break
1419
1420                 case 'Ubuntu16.04':
1421                     assert !isFlowJob
1422                     assert scenario != 'innerloop'
1423                     // Distinguish with the other architectures (arm and x86)
1424                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1425                     break
1426
1427                 case 'Fedora24':
1428                 case 'Ubuntu16.10':
1429                     assert !isFlowJob
1430                     assert scenario != 'innerloop'
1431                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1432                     break
1433
1434                 case 'Ubuntu':
1435                     if (scenario == 'illink') {
1436                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1437                         break
1438                     }
1439                     // fall through
1440
1441                 case 'OSX10.12':
1442                     // Triggers on the non-flow jobs aren't necessary here
1443                     // Corefx testing uses non-flow jobs.
1444                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1445                         break
1446                     }
1447                     switch (scenario) {
1448                         case 'innerloop':
1449                             // PR Triggered jobs. These jobs will run pri0 tests.
1450                             if (configuration == 'Checked') {
1451                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1452                             }
1453                             break
1454
1455                         case 'normal':
1456                             // OSX uses checked for default PR tests
1457                             if (configuration == 'Checked') {
1458                                 // Default trigger
1459                                 assert !job.name.contains("centos")
1460                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1461                             }
1462                             break
1463
1464                         case 'jitdiff':
1465                             if (configuration == 'Checked') {
1466                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1467                             }
1468                             break
1469
1470                         case 'ilrt':
1471                             if (configuration == 'Release') {
1472                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1473                             }
1474                             break
1475
1476                         case 'longgc':
1477                             if (configuration == 'Release') {
1478                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1479                             }
1480                             break
1481
1482                         case 'gcsimulator':
1483                             if (configuration == 'Release') {
1484                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1485                             }
1486                             break
1487
1488                         case 'standalone_gc':
1489                             if (configuration == 'Release' || configuration == 'Checked') {
1490                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1491                             }
1492                             break
1493
1494                         case 'gc_reliability_framework':
1495                             if (configuration == 'Release' || configuration == 'Checked') {
1496                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1497                             }
1498                             break
1499
1500                         default:
1501                             if (isJitStressScenario(scenario)) {
1502                                 def displayStr = getStressModeDisplayName(scenario)
1503                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1504                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1505                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1506                             }
1507                             else if (isR2RScenario(scenario)) {
1508                                 if (configuration == 'Release' || configuration == 'Checked') {
1509                                     def displayStr = getR2RDisplayName(scenario)
1510                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1511                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1512                                 }
1513                             }
1514                             else {
1515                                 println("Unknown scenario: ${scenario}");
1516                                 assert false
1517                             }
1518                             break
1519
1520                     }
1521                     break
1522
1523                 case 'CentOS7.1':
1524                     switch (scenario) {
1525                         case 'innerloop':
1526                             // CentOS uses checked for default PR tests while debug is build only
1527                             if (configuration == 'Debug') {
1528                                 // Default trigger
1529                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1530                             }
1531
1532                             // Make sure this is a flow job to get build and test.
1533                             if (configuration == 'Checked' && isFlowJob) {
1534                                 assert job.name.contains("flow")
1535                                 // Default trigger
1536                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1537                             }
1538                             break
1539
1540                         case 'normal':
1541                             // Make sure this is a flow job to get build and test.
1542                             if (configuration == 'Checked' && isFlowJob) {
1543                                 assert job.name.contains("flow")
1544                                 // Default trigger
1545                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1546                             }
1547                             break
1548
1549                         default:
1550                             if (isR2RScenario(scenario)) {
1551                                 if (configuration == 'Release' || configuration == 'Checked') {
1552                                     def displayStr = getR2RDisplayName(scenario)
1553                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1554                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1555                                 }
1556                             }
1557                             break
1558
1559                     }
1560                     break
1561
1562                 case 'Windows_NT':
1563                     switch (scenario) {
1564                         case 'innerloop':
1565                             // Default trigger
1566                             if (configuration == 'Checked' || configuration == 'Release') {
1567                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1568                             }
1569                             break
1570
1571                         case 'normal':
1572                             if (configuration == 'Checked') {
1573                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1574                             }
1575                             break
1576
1577                         case 'jitdiff':
1578                             if (configuration == 'Checked') {
1579                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1580                             }
1581                             break
1582
1583                         case 'ilrt':
1584                             if (configuration == 'Release') {
1585                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1586                             }
1587                             break
1588
1589                         case 'longgc':
1590                             if (configuration == 'Release') {
1591                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1592                             }
1593                             break
1594
1595                         case 'gcsimulator':
1596                             if (configuration == 'Release') {
1597                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1598                             }
1599                             break
1600
1601                         case 'standalone_gc':
1602                             if (configuration == 'Release' || configuration == 'Checked') {
1603                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1604                             }
1605                             break
1606
1607                         case 'gc_reliability_framework':
1608                             if (configuration == 'Release' || configuration == 'Checked') {
1609                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1610                             }
1611                             break
1612
1613                         case 'illink':
1614                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1615                             break
1616
1617                         default:
1618                             if (isJitStressScenario(scenario)) {
1619                                 def displayStr = getStressModeDisplayName(scenario)
1620                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1621                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1622                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1623                             }
1624                             else if (isR2RScenario(scenario)) {
1625                                 if (configuration == 'Release' || configuration == 'Checked') {
1626                                     def displayStr = getR2RDisplayName(scenario)
1627                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1628                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1629                                 }
1630                             }
1631                             else {
1632                                 println("Unknown scenario: ${scenario}");
1633                                 assert false
1634                             }
1635                             break
1636
1637                     }
1638                     break
1639
1640                 default:
1641                     println("Unknown os: ${os}");
1642                     assert false
1643                     break
1644
1645             }
1646
1647             break
1648         // editor brace matching: }
1649
1650         case 'armem': // editor brace matching: {
1651             job.with {
1652                 publishers {
1653                     azureVMAgentPostBuildAction {
1654                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1655                     }
1656                 }
1657             }
1658
1659             switch (os) {
1660                 case 'Ubuntu':
1661                 case 'Ubuntu16.04':
1662                     assert scenario != 'innerloop'
1663                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1664                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1665                     break
1666
1667                 case 'Tizen':
1668                     architecture = 'armel'
1669
1670                     if (scenario == 'innerloop') {
1671                         if (configuration == 'Checked') {
1672                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1673                         }
1674                     }
1675                     else {
1676                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1677                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1678                     }
1679                     break
1680             }
1681
1682             break
1683         // editor brace matching: }
1684
1685         case 'armlb':
1686         case 'arm': // editor brace matching: {
1687
1688             // Triggers on the non-flow jobs aren't necessary
1689             if (!isFlowJob) {
1690                 break
1691             }
1692
1693             // Set up a private trigger
1694             def contextString = "${os} ${architecture} Cross ${configuration}"
1695             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1696             if (scenario == 'innerloop') {
1697                 contextString += " Innerloop"
1698                 triggerString += "\\W+Innerloop"
1699             }
1700             else {
1701                 contextString += " ${scenario}"
1702                 triggerString += "\\W+${scenario}"
1703             }
1704
1705             if (configuration == 'Debug') {
1706                 contextString += " Build"
1707                 triggerString += "\\W+Build"
1708             } else {
1709                 contextString += " Build and Test"
1710                 triggerString += "\\W+Build and Test"
1711             }
1712
1713             triggerString += ".*"
1714
1715             switch (os) {
1716                 case 'Ubuntu':
1717                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1718                         break
1719                     }
1720
1721                     if (scenario == 'innerloop') {
1722                         if (configuration == 'Checked') {
1723                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1724                         }
1725                     }
1726                     else {
1727                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1728                     }
1729                     break
1730
1731                 case 'Windows_NT':
1732                     if (architecture == "armlb") {
1733                         // Disable armlb windows jobs
1734                         break
1735                     }
1736                     switch (scenario) {
1737                         case 'innerloop':
1738                             // Only Checked is an innerloop trigger.
1739                             if (configuration == 'Checked')
1740                             {
1741                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1742                             }
1743                             break
1744                         case 'normal':
1745                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1746                             break
1747                         default:
1748                             // Stress jobs will use this code path.
1749                             if (isArmWindowsScenario(scenario)) {
1750                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1751                             }
1752                             break
1753                     }
1754                     break
1755                 default:
1756                     println("NYI os: ${os}");
1757                     assert false
1758                     break
1759             }
1760             break
1761         // editor brace matching: }
1762         case 'arm64': // editor brace matching: {
1763             // Set up a private trigger
1764             def contextString = "${os} ${architecture} Cross ${configuration}"
1765             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1766
1767             if (scenario == 'innerloop') {
1768                 contextString += " Innerloop"
1769                 triggerString += "\\W+Innerloop"
1770             }
1771             else {
1772                 contextString += " ${scenario}"
1773                 triggerString += "\\W+${scenario}"
1774             }
1775
1776             if (configuration == 'Debug') {
1777                 contextString += " Build"
1778                 triggerString += "\\W+Build"
1779             } else {
1780                 contextString += " Build and Test"
1781                 triggerString += "\\W+Build and Test"
1782             }
1783
1784             triggerString += ".*"
1785
1786             switch (os) {
1787                 case 'Ubuntu':
1788                 case 'Ubuntu16.04':
1789                     switch (scenario) {
1790                         case 'innerloop':
1791                             if (configuration == 'Debug' && !isFlowJob) {
1792                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1793                             }
1794
1795                             break
1796                         case 'normal':
1797                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1798                             break
1799                         default:
1800                             if (isR2RScenario(scenario)) {
1801                                 if (configuration == 'Checked' || configuration == 'Release') {
1802                                     def displayStr = getR2RDisplayName(scenario)
1803                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1804                                 }
1805                             }
1806                             break
1807                     }
1808                     break
1809
1810                 case 'Windows_NT':
1811                     // Triggers on the non-flow jobs aren't necessary here
1812                     if (!isFlowJob) {
1813                         break
1814                     }
1815
1816                     assert isArmWindowsScenario(scenario)
1817                     switch (scenario) {
1818                         case 'innerloop':
1819                             if (configuration == 'Checked') {
1820                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1821                             }
1822
1823                             break
1824                         case 'normal':
1825                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1826                             break
1827                         default:
1828                             // Stress jobs will use this code path.
1829                             if (isArmWindowsScenario(scenario)) {
1830                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1831                             }
1832                             break
1833                     }
1834                     break
1835                 default:
1836                     println("NYI os: ${os}");
1837                     assert false
1838                     break
1839             }
1840             break
1841
1842         // editor brace matching: }
1843         case 'x86': // editor brace matching: {
1844             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1845             if (os == 'Ubuntu') {
1846                 // Triggers on the non-flow jobs aren't necessary here
1847                 if (!isFlowJob) {
1848                     break
1849                 }
1850
1851                 // on-demand only for ubuntu x86
1852                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1853                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1854                 break
1855
1856             }
1857             switch (scenario) {
1858                 case 'innerloop':
1859                     if (configuration == 'Checked' || configuration == 'Release') {
1860                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1861                     }
1862                     break
1863
1864                 case 'normal':
1865                     if (configuration == 'Checked') {
1866                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1867                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1868                     }
1869                     break
1870
1871                 case 'ilrt':
1872                     if (configuration == 'Release') {
1873                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1874                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1875                     }
1876                     break
1877
1878                 case 'longgc':
1879                     if (configuration == 'Release') {
1880                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1881                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1882                     }
1883                     break
1884
1885                 case 'gcsimulator':
1886                     if (configuration == 'Release') {
1887                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1888                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1889                     }
1890                     break
1891
1892                 case 'standalone_gc':
1893                     if (configuration == 'Release' || configuration == 'Checked') {
1894                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1895                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1896                     }
1897                     break
1898
1899                 case 'illink':
1900                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1901                     break
1902
1903                 default:
1904                     if (isJitStressScenario(scenario)) {
1905                         def displayStr = getStressModeDisplayName(scenario)
1906                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1907                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1908                     }
1909                     else if (isR2RScenario(scenario)) {
1910                         if (configuration == 'Release' || configuration == 'Checked') {
1911                             def displayStr = getR2RDisplayName(scenario)
1912                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1913                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1914                         }
1915                     }
1916                     else {
1917                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1918                         assert false
1919                     }
1920                     break
1921
1922             }
1923             break
1924
1925          // editor brace matching: }
1926         case 'x64_arm64_altjit':
1927         case 'x86_arm_altjit': // editor brace matching: {
1928             assert (os == 'Windows_NT')
1929             switch (scenario) {
1930                 case 'normal':
1931                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1932                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1933                     break
1934                 default:
1935                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1936                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1937                     break
1938             }
1939             break
1940
1941         // editor brace matching: }
1942         default:
1943             println("Unknown architecture: ${architecture}");
1944             assert false
1945             break
1946     }
1947 }
1948
1949 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1950     def buildCommands = []
1951     def osGroup = getOSGroup(os)
1952     def lowerConfiguration = configuration.toLowerCase()
1953
1954     def priority = '1'
1955     if (scenario == 'innerloop') {
1956         priority = '0'
1957     }
1958
1959     def doCoreFxTesting = isCoreFxScenario(scenario)
1960
1961     // Calculate the build steps, archival, and xunit results
1962     switch (os) {
1963         case 'Windows_NT': // editor brace matching: {
1964             switch (architecture) {
1965                 case 'x64':
1966                 case 'x86':
1967                 case 'x86_arm_altjit':
1968                 case 'x64_arm64_altjit':
1969                     def arch = architecture
1970                     def buildOpts = ''
1971                     if (architecture == 'x86_arm_altjit') {
1972                         arch = 'x86'
1973                     }
1974                     else if (architecture == 'x64_arm64_altjit') {
1975                         arch = 'x64'
1976                     }
1977
1978                     if (scenario == 'formatting') {
1979                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1980                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1981                         break
1982                     }
1983
1984                     if (scenario == 'illink') {
1985                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1986                     }
1987
1988                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
1989                     if ((lowerConfiguration == 'release') &&
1990                         (scenario in Constants.basicScenarios) &&
1991                         (architecture != 'x86_arm_altjit') &&
1992                         (architecture != 'x64_arm64_altjit')) {
1993
1994                         buildOpts += ' -enforcepgo'
1995                     }
1996
1997                     if (doCoreFxTesting) {
1998                         buildOpts += ' skiptests';
1999                     } else {
2000                         buildOpts += " -priority=${priority}"
2001                     }
2002
2003                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2004                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2005                     // 35 characters long.
2006
2007                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2008
2009                     if (!isBuildOnly) {
2010                         def runtestArguments = ''
2011                         def testOpts = 'collectdumps'
2012
2013                         if (isR2RScenario(scenario)) {
2014
2015                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2016                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2017                             // to cause the tests to be crossgen'ed.
2018
2019                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2020                                 testOpts += ' crossgenaltjit protononjit.dll'
2021                             } else {
2022                                 testOpts += ' crossgen'
2023                             }
2024
2025                             testOpts += ' runcrossgentests'
2026                         }
2027                         else if (scenario == 'jitdiff') {
2028                             testOpts += ' jitdisasm crossgen'
2029                         }
2030                         else if (scenario == 'ilrt') {
2031                             testOpts += ' ilasmroundtrip'
2032                         }
2033                         else if (isLongGc(scenario)) {
2034                             testOpts += " ${scenario} sequential"
2035                         }
2036                         else if (scenario == 'standalone_gc') {
2037                             testOpts += ' gcname clrgc.dll'
2038                         }
2039                         else if (scenario == 'illink') {
2040                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2041                         }
2042
2043                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2044                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2045                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2046                         // which is read by the xunit harness.)
2047                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2048                         {
2049                             def timeout = 1800000
2050                             testOpts += " timeout ${timeout}"
2051                         }
2052
2053                         // If we are running a stress mode, we should write out the set of key
2054                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2055
2056                         def envScriptPath = ''
2057                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2058                             def buildCommandsStr = ''
2059                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2060                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2061
2062                             if (isJitStressScenario(scenario)) {
2063                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2064                             }
2065                             else if (isR2RStressScenario(scenario)) {
2066                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2067                             }
2068
2069                             if (architecture == 'x86_arm_altjit') {
2070                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2071                             }
2072                             else if (architecture == 'x64_arm64_altjit') {
2073                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2074                             }
2075
2076                             envScriptFinalize(os, envScriptPath)
2077
2078                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2079                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2080                             // into a single script.
2081                             buildCommands += buildCommandsStr
2082                         }
2083                         else if (architecture == 'x86_arm_altjit') {
2084                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2085                         }
2086                         else if (architecture == 'x64_arm64_altjit') {
2087                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2088                         }
2089                         if (envScriptPath != '') {
2090                             testOpts += " TestEnv ${envScriptPath}"
2091                         }
2092
2093                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2094
2095                         if (doCoreFxTesting) {
2096                             def workspaceRelativeFxRoot = "_/fx"
2097                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2098
2099                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2100
2101                             // Archive and process (only) the test results
2102                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2103                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2104
2105                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2106                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2107                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2108                         }
2109                         else if (isGcReliabilityFramework(scenario)) {
2110                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2111                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2112                         }
2113                         else {
2114                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2115                         }
2116                     } // end if (!isBuildOnly)
2117
2118                     if (!doCoreFxTesting) {
2119                         // Run the rest of the build
2120                         // Build the mscorlib for the other OS's
2121                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2122                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2123
2124                         if (arch == 'x64') {
2125                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2126                         }
2127
2128                         // Zip up the tests directory so that we don't use so much space/time copying
2129                         // 10s of thousands of files around.
2130                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2131
2132                         if (!isJitStressScenario(scenario)) {
2133                             // For Windows, pull full test results and test drops for x86/x64.
2134                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2135                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2136                         }
2137
2138                         if (scenario == 'jitdiff') {
2139                             // retrieve jit-dasm output for base commit, and run jit-diff
2140                             if (!isBuildOnly) {
2141                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2142                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2143                             }
2144                         }
2145
2146                         if (!isBuildOnly) {
2147                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2148                         }
2149                     }
2150                     break
2151                 case 'armlb':
2152                 case 'arm':
2153                     assert isArmWindowsScenario(scenario)
2154
2155                     def buildArchitecture = 'arm'
2156
2157                     def buildOpts = ''
2158
2159                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2160                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2161                     if (architecture == 'armlb') {
2162                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2163                     }
2164
2165                     if (doCoreFxTesting) {
2166                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2167                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2168                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2169                         //
2170                         // buildOpts += ' skiptests';
2171                         buildOpts += " -priority=0"
2172                     } else {
2173                         buildOpts += " -priority=${priority}"
2174                     }
2175
2176                     // This is now a build only job. Do not run tests. Use the flow job.
2177                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2178
2179                     if (doCoreFxTesting) {
2180                         assert isBuildOnly
2181                         assert architecture == 'arm'
2182
2183                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2184                         // to determine the correct matching corefx version git commit hash.
2185                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2186
2187                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2188                         def envScriptPath = ''
2189                         def buildCommandsStr = ''
2190                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2191                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2192                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2193                         envScriptFinalize(os, envScriptPath)
2194                         buildCommands += buildCommandsStr
2195
2196                         def workspaceRelativeFxRootLinux = "_/fx"
2197                         def workspaceRelativeFxRootWin = "_\\fx"
2198                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2199
2200                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2201
2202                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2203                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2204                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2205
2206                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2207                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2208                     } else {
2209                         // Zip up the tests directory so that we don't use so much space/time copying
2210                         // 10s of thousands of files around.
2211                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2212
2213                         // Add archival.
2214                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2215                     }
2216                     break
2217                 case 'arm64':
2218                     assert isArmWindowsScenario(scenario)
2219
2220                     // This is now a build only job. Do not run tests. Use the flow job.
2221                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2222
2223                     // Zip up the tests directory so that we don't use so much space/time copying
2224                     // 10s of thousands of files around.
2225                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2226
2227                     // Add archival.
2228                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2229                     break
2230                 default:
2231                     println("Unknown architecture: ${architecture}");
2232                     assert false
2233                     break
2234             }
2235             break
2236         // end case 'Windows_NT'; editor brace matching: }
2237         case 'Ubuntu':
2238         case 'Ubuntu16.04':
2239         case 'Ubuntu16.10':
2240         case 'Debian8.4':
2241         case 'OSX10.12':
2242         case 'CentOS7.1':
2243         case 'RHEL7.2':
2244         case 'Tizen':
2245         case 'Fedora24': // editor brace matching: {
2246             switch (architecture) {
2247                 case 'x64':
2248                 case 'x86':
2249                     if (architecture == 'x86' && os == 'Ubuntu') {
2250                         // build and PAL test
2251                         def dockerImage = getDockerImageName(architecture, os, true)
2252                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2253                         dockerImage = getDockerImageName(architecture, os, false)
2254                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2255                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2256                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2257                         break
2258                     }
2259
2260                     if (scenario == 'formatting') {
2261                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2262                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2263                         break
2264                     }
2265
2266                     if (scenario == 'illink') {
2267                         assert(os == 'Ubuntu')
2268                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2269                     }
2270
2271                     if (!doCoreFxTesting) {
2272                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2273                         // only on supported OS platforms.
2274                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2275                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2276
2277                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2278                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2279
2280                         // Basic archiving of the build
2281                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2282                         // And pal tests
2283                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2284                     }
2285                     else {
2286                         // Corefx stress testing
2287                         assert os == 'Ubuntu'
2288                         assert architecture == 'x64'
2289                         assert lowerConfiguration == 'checked'
2290                         assert isJitStressScenario(scenario)
2291
2292                         // Build coreclr
2293                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2294
2295                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2296
2297                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2298                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2299                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2300                         buildCommands += envScriptCmds
2301
2302                         // Build and text corefx
2303                         def workspaceRelativeFxRoot = "_/fx"
2304                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2305
2306                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2307
2308                         // Archive and process (only) the test results
2309                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2310                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2311                     }
2312                     break
2313                 case 'arm64':
2314                     if (!doCoreFxTesting) {
2315                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2316
2317                         // HACK -- Arm64 does not have corefx jobs yet.
2318                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2319                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2320                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2321                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2322
2323                         // Basic archiving of the build
2324                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2325                     }
2326                     break
2327                 case 'armem':
2328                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2329                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2330
2331                     // default values for Ubuntu
2332                     def arm_abi = "arm"
2333                     def linuxCodeName = "trusty"
2334                     if (os == 'Ubuntu16.04') {
2335                         linuxCodeName = "xenial"
2336                     }
2337                     else if (os == 'Tizen') {
2338                         arm_abi = "armel"
2339                         linuxCodeName = "tizen"
2340                     }
2341
2342                     // Unzip the Windows test binaries first. Exit with 0
2343                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2344
2345                     // Unpack the corefx binaries
2346                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2347                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2348                     if (os != 'Tizen') {
2349                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2350                     }
2351                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2352                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2353                     if (os == 'Ubuntu') {
2354                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2355                     }
2356
2357                     // Call the ARM CI script to cross build and test using docker
2358                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2359                     --mode=docker \\
2360                     --${arm_abi} \\
2361                     --linuxCodeName=${linuxCodeName} \\
2362                     --buildConfig=${lowerConfiguration} \\
2363                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2364                     --coreFxBinDir=./bin/CoreFxBinDir \\
2365                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2366
2367                     // Basic archiving of the build, no pal tests
2368                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2369                     break
2370                 case 'arm':
2371                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2372                     assert (os == 'Ubuntu')
2373
2374                     // Add some useful information to the log file. Ignore return codes.
2375                     buildCommands += "uname -a || true"
2376
2377                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2378                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2379
2380                     def dockerImage = getDockerImageName(architecture, os, true)
2381                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm ${dockerImage} "
2382
2383                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross"
2384
2385                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2386                     // download the appropriate CoreFX packages.
2387                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2388                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2389
2390                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2391
2392                     // ZIP up for the test job (created in the flow job code):
2393                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2394                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2395                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2396                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2397
2398                     // These commands are assumed to be run from the root of the workspace.
2399                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2400                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2401
2402                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2403                     break
2404                 default:
2405                     println("Unknown architecture: ${architecture}");
2406                     assert false
2407                     break
2408             }
2409             break
2410         // editor brace matching: }
2411         default:
2412             println("Unknown os: ${os}");
2413             assert false
2414             break
2415     } // os
2416
2417     return buildCommands
2418 }
2419
2420 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2421 // Returns true if the job should be generated.
2422 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2423 {
2424     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2425     // All other scenarios do Pri-1 testing.
2426     if (scenario == 'innerloop' && !isPR) {
2427         return false
2428     }
2429
2430     // Tizen is only supported for armem architecture
2431     if (os == 'Tizen' && architecture != 'armem') {
2432         return false
2433     }
2434
2435     // Filter based on architecture.
2436
2437     switch (architecture) {
2438         case 'arm64':
2439         case 'arm':
2440             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2441                 return false
2442             }
2443             break
2444         case 'armem':
2445             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2446                 return false
2447             }
2448             break
2449         case 'armlb':
2450             // Do not create armlb jobs
2451             return false
2452         case 'x86_arm_altjit':
2453         case 'x64_arm64_altjit':
2454             if (os != 'Windows_NT') {
2455                 return false
2456             }
2457             break
2458         case 'x86':
2459             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2460                 return false
2461             }
2462             break
2463         case 'x64':
2464             // Everything implemented
2465             break
2466         default:
2467             println("Unknown architecture: ${architecture}")
2468             assert false
2469             break
2470     }
2471
2472     // Which (Windows) build only jobs are required?
2473
2474     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2475
2476     if (isBuildOnly) {
2477         switch (architecture) {
2478             case 'arm':
2479                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2480                 if (!isCoreFxScenario(scenario)) {
2481                     return false
2482                 }
2483                 break
2484             case 'x64':
2485             case 'x86':
2486                 if (!isNormalOrInnerloop) {
2487                     return false
2488                 }
2489                 break
2490             default:
2491                 return false
2492         }
2493     }
2494
2495     // Filter based on scenario.
2496
2497     if (isJitStressScenario(scenario)) {
2498         if (configuration != 'Checked') {
2499             return false
2500         }
2501
2502         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2503         if (!isEnabledOS) {
2504             return false
2505         }
2506
2507         switch (architecture) {
2508             case 'x64':
2509             case 'x86_arm_altjit':
2510             case 'x64_arm64_altjit':
2511                 break
2512
2513             case 'x86':
2514                 // x86 ubuntu: no stress modes
2515                 if (os == 'Ubuntu') {
2516                     return false
2517                 }
2518                 break
2519
2520             case 'arm':
2521                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2522                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2523                     return false
2524                 }
2525                 break
2526
2527             default:
2528                 // arm64, armlb: stress is handled through flow jobs.
2529                 // armem: no stress jobs for ARM emulator.
2530                 return false
2531         }
2532     }
2533     else if (isR2RScenario(scenario)) {
2534         if (os != 'Windows_NT') {
2535             return false
2536         }
2537         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2538         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2539             return false
2540         }
2541     }
2542     else {
2543         // Skip scenarios
2544         switch (scenario) {
2545             case 'ilrt':
2546                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2547                 // the default scenario build
2548                 if (os != 'Windows_NT') {
2549                     return false
2550                 }
2551                 // Only x64 for now
2552                 if (architecture != 'x64') {
2553                     return false
2554                 }
2555                 // Release only
2556                 if (configuration != 'Release') {
2557                     return false
2558                 }
2559                 break
2560             case 'jitdiff':
2561                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2562                     return false
2563                 }
2564                 if (architecture != 'x64') {
2565                     return false
2566                 }
2567                 if (configuration != 'Checked') {
2568                     return false
2569                 }
2570                 break
2571             case 'longgc':
2572             case 'gcsimulator':
2573                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2574                     return false
2575                 }
2576                 if (architecture != 'x64') {
2577                     return false
2578                 }
2579                 if (configuration != 'Release') {
2580                     return false
2581                 }
2582                 break
2583             case 'gc_reliability_framework':
2584             case 'standalone_gc':
2585                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2586                     return false
2587                 }
2588
2589                 if (architecture != 'x64') {
2590                     return false
2591                 }
2592
2593                 if (configuration != 'Release' && configuration != 'Checked') {
2594                     return false
2595                 }
2596                 break
2597             // We only run Windows and Ubuntu x64 Checked for formatting right now
2598             case 'formatting':
2599                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2600                     return false
2601                 }
2602                 if (architecture != 'x64') {
2603                     return false
2604                 }
2605                 if (configuration != 'Checked') {
2606                     return false
2607                 }
2608                 break
2609             case 'illink':
2610                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2611                     return false
2612                 }
2613                 if (architecture != 'x64' && architecture != 'x86') {
2614                     return false
2615                 }
2616                 break
2617             case 'normal':
2618                 // Nothing skipped
2619                 break
2620             case 'innerloop':
2621                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2622                     return false
2623                 }
2624                 break
2625             default:
2626                 println("Unknown scenario: ${scenario}")
2627                 assert false
2628                 break
2629         }
2630     }
2631
2632     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2633     // runtime behavior, not compile-time behavior, are not interesting.
2634     switch (architecture) {
2635         case 'x86_arm_altjit':
2636         case 'x64_arm64_altjit':
2637             if (isGCStressRelatedTesting(scenario)) {
2638                 return false
2639             }
2640             break
2641         default:
2642             break
2643     }
2644
2645     // The job was not filtered out, so we should generate it!
2646     return true
2647 }
2648
2649 Constants.allScenarios.each { scenario ->
2650     [true, false].each { isPR ->
2651         Constants.architectureList.each { architecture ->
2652             Constants.configurationList.each { configuration ->
2653                 Constants.osList.each { os ->
2654                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2655                     // and reset the os to Windows_NT
2656                     def isBuildOnly = false
2657                     if (os == 'Windows_NT_BuildOnly') {
2658                         isBuildOnly = true
2659                         os = 'Windows_NT'
2660                     }
2661
2662                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2663                         return
2664                     }
2665
2666                     // Calculate names
2667                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2668                     def folderName = getJobFolder(scenario)
2669
2670                     // Create the new job
2671                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2672                     addToViews(newJob, isPR, architecture, os)
2673
2674                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2675
2676                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2677                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2678                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2679
2680                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2681                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2682                     // artifacts produced.
2683                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2684                         // Define the Windows Tests and Corefx build job names
2685                         def lowerConfiguration = configuration.toLowerCase()
2686                         def WindowsTestsName = projectFolder + '/' +
2687                                                Utilities.getFullJobName(project,
2688                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2689                                                                         false)
2690                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2691                                            Utilities.getFolderName(branch)
2692
2693                         def arm_abi = 'arm'
2694                         def corefx_os = 'linux'
2695                         if (os == 'Tizen') {
2696                             arm_abi = 'armel'
2697                             corefx_os = 'tizen'
2698                         }
2699
2700                         // Let's use release CoreFX to test checked CoreCLR,
2701                         // because we do not generate checked CoreFX in CoreFX CI yet.
2702                         def corefx_lowerConfiguration = lowerConfiguration
2703                         if (lowerConfiguration == 'checked') {
2704                             corefx_lowerConfiguration = 'release'
2705                         }
2706
2707                         // Copy the Windows test binaries and the Corefx build binaries
2708                         newJob.with {
2709                             steps {
2710                                 copyArtifacts(WindowsTestsName) {
2711                                     includePatterns('bin/tests/tests.zip')
2712                                     buildSelector {
2713                                         latestSuccessful(true)
2714                                     }
2715                                 }
2716                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2717                                     includePatterns('bin/build.tar.gz')
2718                                     buildSelector {
2719                                         latestSuccessful(true)
2720                                     }
2721                                 }
2722                             } // steps
2723                         } // newJob.with
2724                     }
2725
2726                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2727
2728                     newJob.with {
2729                         steps {
2730                             if (os == 'Windows_NT') {
2731                                 buildCommands.each { buildCommand ->
2732                                     batchFile(buildCommand)
2733                                 }
2734                             }
2735                             else {
2736                                 buildCommands.each { buildCommand ->
2737                                     shell(buildCommand)
2738                                 }
2739                             }
2740                         } // steps
2741                     } // newJob.with
2742
2743                 } // os
2744             } // configuration
2745         } // architecture
2746     } // isPR
2747 } // scenario
2748
2749 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2750 // Returns the newly created job.
2751 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2752 {
2753     def osGroup = getOSGroup(os)
2754     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2755
2756     def jobFolder = getJobFolder(scenario)
2757     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2758         parameters {
2759             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2760         }
2761
2762         steps {
2763             // Set up the copies
2764
2765             // Coreclr build we are trying to test
2766             //
2767             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2768
2769             copyArtifacts(inputCoreCLRBuildName) {
2770                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2771                 buildSelector {
2772                     buildNumber('${CORECLR_BUILD}')
2773                 }
2774             }
2775
2776             if (isCoreFxScenario(scenario)) {
2777
2778                 // Only arm supported for corefx testing now.
2779                 assert architecture == 'arm'
2780
2781                 // Unzip CoreFx runtime
2782                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2783
2784                 // Unzip CoreFx tests.
2785                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2786
2787                 // Add the script to run the corefx tests
2788                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2789                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2790                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2791                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2792
2793             } else { // !isCoreFxScenario(scenario)
2794
2795                 // Unzip tests.
2796                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2797
2798                 def buildCommands = ""
2799
2800                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2801                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2802                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2803
2804                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2805                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2806
2807                 // For all jobs
2808                 addEnvVariable("CORE_ROOT", coreRootLocation)
2809                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2810                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2811
2812                 // ARM legacy backend; this is an altjit.
2813                 if (architecture == 'armlb') {
2814                     addEnvVariable("COMPlus_AltJit", "*")
2815                     addEnvVariable("COMPlus_AltJitNgen", "*")
2816                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2817                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2818                 }
2819
2820                 // If we are running a stress mode, we'll set those variables as well
2821                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2822                     def stressValues = null
2823                     if (isJitStressScenario(scenario)) {
2824                         stressValues = Constants.jitStressModeScenarios[scenario]
2825                     }
2826                     else {
2827                         stressValues = Constants.r2rStressScenarios[scenario]
2828                     }
2829
2830                     stressValues.each { key, value ->
2831                         addEnvVariable(key, value)
2832                     }
2833                 }
2834
2835                 if (isR2RScenario(scenario)) {
2836                     // Crossgen the framework assemblies.
2837                     buildCommands += """
2838 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2839 @goto skip_PrecompileAssembly
2840
2841 :PrecompileAssembly
2842 @REM Skip mscorlib since it is already precompiled.
2843 @if /I "%3" == "mscorlib.dll" exit /b 0
2844 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2845
2846 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2847 @if "%errorlevel%" == "-2146230517" (
2848     echo %2 is not a managed assembly.
2849 ) else if "%errorlevel%" == "-2146234344" (
2850     echo %2 is not a managed assembly.
2851 ) else if %errorlevel% neq 0 (
2852     echo Unable to precompile %2
2853 ) else (
2854     echo Precompiled %2
2855 )
2856 @exit /b 0
2857
2858 :skip_PrecompileAssembly
2859 """
2860
2861                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2862                     // crossgen on tests before running them.
2863                     addEnvVariable("RunCrossGen", "true")
2864                 } // isR2RScenario(scenario)
2865
2866                 // Create the smarty command
2867                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2868                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2869                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2870                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2871
2872                 if (architecture == 'armlb') {
2873                     addExclude("LEGACYJIT_FAIL")
2874                 }
2875
2876                 // Exclude tests based on scenario.
2877                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2878                     addArchSpecificExclude(architecture, excludeTag)
2879                 }
2880
2881                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2882                 if (scenario == 'innerloop') {
2883                     addExclude("pri1")
2884                 }
2885
2886                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2887                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2888                 addExclude("LONG_RUNNING")
2889
2890                 smartyCommand += "/lstFile Tests.lst"
2891
2892                 def testListArch = [
2893                     'arm64': 'arm64',
2894                     'arm': 'arm',
2895                     'armlb': 'arm'
2896                 ]
2897
2898                 def archLocation = testListArch[architecture]
2899
2900                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2901                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2902                 addCommand("${smartyCommand}")
2903
2904                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2905                 // However, we also need to remove all the variables that were set during this batch file, so we
2906                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2907                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2908                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2909                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2910                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2911                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2912
2913                 // ZIP up the smarty output, no matter what the smarty result.
2914                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2915
2916                 addCommand("echo %errorlevel%")
2917                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2918
2919                 // Use the smarty errorlevel as the script errorlevel.
2920                 addCommand("exit /b %__save_smarty_errorlevel%")
2921
2922                 batchFile(buildCommands)
2923             } // non-corefx testing
2924         } // steps
2925     } // job
2926
2927     if (!isCoreFxScenario(scenario)) {
2928         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2929
2930         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2931         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2932         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2933         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2934     }
2935
2936     return newJob
2937 }
2938
2939 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2940 // E.g., non-Windows tests.
2941 // Returns the newly created job.
2942 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2943 {
2944     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2945
2946     def osGroup = getOSGroup(os)
2947     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2948
2949     def testOpts = ''
2950     def useServerGC = false
2951
2952     // Enable Server GC for Ubuntu PR builds
2953     // REVIEW: why? Does this apply to all architectures? Why only PR?
2954     if (os == 'Ubuntu' && isPR) {
2955         testOpts += ' --useServerGC'
2956         useServerGC = true
2957     }
2958
2959     if (isR2RScenario(scenario)) {
2960
2961         testOpts += ' --crossgen --runcrossgentests'
2962
2963         if (scenario == 'r2r_jitstress1') {
2964             testOpts += ' --jitstress=1'
2965         }
2966         else if (scenario == 'r2r_jitstress2') {
2967             testOpts += ' --jitstress=2'
2968         }
2969         else if (scenario == 'r2r_jitstressregs1') {
2970             testOpts += ' --jitstressregs=1'
2971         }
2972         else if (scenario == 'r2r_jitstressregs2') {
2973             testOpts += ' --jitstressregs=2'
2974         }
2975         else if (scenario == 'r2r_jitstressregs3') {
2976             testOpts += ' --jitstressregs=3'
2977         }
2978         else if (scenario == 'r2r_jitstressregs4') {
2979             testOpts += ' --jitstressregs=4'
2980         }
2981         else if (scenario == 'r2r_jitstressregs8') {
2982             testOpts += ' --jitstressregs=8'
2983         }
2984         else if (scenario == 'r2r_jitstressregs0x10') {
2985             testOpts += ' --jitstressregs=0x10'
2986         }
2987         else if (scenario == 'r2r_jitstressregs0x80') {
2988             testOpts += ' --jitstressregs=0x80'
2989         }
2990         else if (scenario == 'r2r_jitstressregs0x1000') {
2991             testOpts += ' --jitstressregs=0x1000'
2992         }
2993         else if (scenario == 'r2r_jitminopts') {
2994             testOpts += ' --jitminopts'
2995         }
2996         else if (scenario == 'r2r_jitforcerelocs') {
2997             testOpts += ' --jitforcerelocs'
2998         }
2999         else if (scenario == 'r2r_gcstress15') {
3000             testOpts += ' --gcstresslevel=0xF'
3001         }
3002     }
3003     else if (scenario == 'jitdiff') {
3004         testOpts += ' --jitdisasm --crossgen'
3005     }
3006     else if (scenario == 'illink') {
3007         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3008     }
3009     else if (isLongGc(scenario)) {
3010         // Long GC tests behave very poorly when they are not
3011         // the only test running (many of them allocate until OOM).
3012         testOpts += ' --sequential'
3013
3014         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3015         // for running long GC and GCSimulator tests, respectively. We don't use them
3016         // here because using a playlist file produces much more readable output on the CI machines
3017         // and reduces running time.
3018         //
3019         // The Long GC playlist contains all of the tests that are
3020         // going to be run. The GCSimulator playlist contains all of
3021         // the GC simulator tests.
3022         if (scenario == 'longgc') {
3023             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3024         }
3025         else if (scenario == 'gcsimulator') {
3026             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3027         }
3028     }
3029     else if (isGcReliabilityFramework(scenario)) {
3030         testOpts += ' --build-overlay-only'
3031     }
3032     else if (scenario == 'standalone_gc') {
3033         if (osGroup == 'OSX') {
3034             testOpts += ' --gcname=libclrgc.dylib'
3035         }
3036         else if (osGroup == 'Linux') {
3037             testOpts += ' --gcname=libclrgc.so'
3038         }
3039         else {
3040             println("Unexpected OS group: ${osGroup} for os ${os}")
3041             assert false
3042         }
3043     }
3044
3045     def jobFolder = getJobFolder(scenario)
3046     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3047         parameters {
3048             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3049             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3050         }
3051
3052         steps {
3053             // Set up the copies
3054
3055             // Coreclr build containing the tests and mscorlib
3056             // pri1 jobs still need to copy windows_nt built tests
3057             assert inputTestsBuildName != null
3058             copyArtifacts(inputTestsBuildName) {
3059                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3060                 buildSelector {
3061                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3062                 }
3063             }
3064
3065             // Coreclr build we are trying to test
3066             //
3067             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3068
3069             copyArtifacts(inputCoreCLRBuildName) {
3070                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3071                 buildSelector {
3072                     buildNumber('${CORECLR_BUILD}')
3073                 }
3074             }
3075
3076             if (isUbuntuArmJob) {
3077                 // Add some useful information to the log file. Ignore return codes.
3078                 shell("uname -a || true")
3079             }
3080
3081             if (architecture == 'arm64') {
3082                 shell("mkdir -p ./bin/CoreFxBinDir")
3083                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3084                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3085             }
3086             else if (architecture == 'x86') {
3087                 shell("mkdir ./bin/CoreFxNative")
3088
3089                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3090
3091                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3092                     includePatterns('bin/build.tar.gz')
3093                     targetDirectory('bin/CoreFxNative')
3094                     buildSelector {
3095                         latestSuccessful(true)
3096                     }
3097                 }
3098
3099                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3100             }
3101
3102             // Unzip the tests first.  Exit with 0
3103             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3104             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3105
3106             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3107             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3108             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3109             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3110             // copied correctly.
3111             if (isUbuntuArmJob) {
3112                 def lowerConfiguration = configuration.toLowerCase()
3113                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3114                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3115             }
3116             else {
3117                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3118             }
3119
3120             // Execute the tests
3121             def runDocker = isNeedDocker(architecture, os, false)
3122             def dockerPrefix = ""
3123             def dockerCmd = ""
3124             if (runDocker) {
3125                 def dockerImage = getDockerImageName(architecture, os, false)
3126                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3127                 dockerCmd = dockerPrefix + "${dockerImage} "
3128             }
3129
3130             // If we are running a stress mode, we'll set those variables first
3131             if (isJitStressScenario(scenario)) {
3132                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3133                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3134                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3135                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3136                 shell("${envScriptCmds}")
3137                 testOpts += " --test-env=${scriptFileName}"
3138             }
3139
3140             // TODO: how to handle GCStress-related testing for Ubuntu/arm?
3141             if (isGCStressRelatedTesting(scenario)) {
3142                 shell('./init-tools.sh')
3143             }
3144
3145             def runScript = ""
3146             if (isUbuntuArmJob) {
3147                 // Use 'runtesttilstable.sh' to rerun failing tests (in sequential mode);
3148                 // there are many tests that pass on rerun (currently), and we don't want
3149                 // that flakiness to affect overall test job robustness.
3150                 runScript = "${dockerCmd}./tests/runtesttilstable.sh"
3151             } else {
3152                 runScript = "${dockerCmd}./tests/runtest.sh"
3153             }
3154
3155             shell("""\
3156 ${runScript} \\
3157     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3158     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3159     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3160     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3161
3162             if (isGcReliabilityFramework(scenario)) {
3163                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3164                 if (useServerGC) {
3165                     if (runDocker) {
3166                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3167                     }
3168                     else {
3169                         shell("export COMPlus_gcServer=1")
3170                     }
3171                 }
3172
3173                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3174             }
3175         } // steps
3176     } // job
3177
3178     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3179     if (os in ['Ubuntu']) {
3180         SummaryBuilder summaries = new SummaryBuilder()
3181         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3182         summaries.emit(newJob)
3183     }
3184
3185     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3186     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3187
3188     return newJob
3189 }
3190
3191 // Create a test job that will be used by a flow job.
3192 // Returns the newly created job.
3193 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3194 {
3195     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3196
3197     def newJob = null
3198     if (windowsArmJob) {
3199         assert inputTestsBuildName == null
3200         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3201     } else {
3202         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3203     }
3204
3205     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3206
3207     addToViews(newJob, isPR, architecture, os)
3208
3209     if (scenario == 'jitdiff') {
3210         def osGroup = getOSGroup(os)
3211         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3212     }
3213
3214     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3215     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3216
3217     return newJob
3218 }
3219
3220 // Create a flow job to tie together a build job with the given test job.
3221 // Returns the new flow job.
3222 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3223 {
3224     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3225         // Do not create the flow job for RHEL jobs.
3226         return
3227     }
3228
3229     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3230     // Linux CoreCLR test
3231     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3232     def jobFolder = getJobFolder(scenario)
3233
3234     def newFlowJob = null
3235
3236     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3237     if (windowsArmJob) {
3238
3239         assert inputTestsBuildName == null
3240
3241         // For Windows arm jobs there is no reason to build a parallel test job.
3242         // The product build supports building and archiving the tests.
3243
3244         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3245                         buildFlow("""\
3246 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3247
3248 // And then build the test build
3249 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3250 """)
3251         }
3252         JobReport.Report.addReference(inputCoreCLRBuildName)
3253         JobReport.Report.addReference(fullTestJobName)
3254     }
3255     else {
3256         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3257                         buildFlow("""\
3258 // Build the input jobs in parallel
3259 parallel (
3260 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3261 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3262 )
3263
3264 // And then build the test build
3265 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3266                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3267 """)
3268         }
3269         JobReport.Report.addReference(inputCoreCLRBuildName)
3270         JobReport.Report.addReference(inputTestsBuildName)
3271         JobReport.Report.addReference(fullTestJobName)
3272     }
3273
3274     addToViews(newFlowJob, isPR, architecture, os)
3275
3276     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3277
3278     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3279     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3280
3281     return newFlowJob
3282 }
3283
3284 // Determine if we should generate a flow job for the given parameters.
3285 // Returns true if the job should be generated.
3286 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3287 {
3288     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3289     // All other scenarios do Pri-1 testing.
3290     if (scenario == 'innerloop' && !isPR) {
3291         return false
3292     }
3293
3294     // Filter based on OS and architecture.
3295
3296     switch (architecture) {
3297         case 'arm64':
3298             if (os != "Ubuntu" && os != "Windows_NT") {
3299                 return false
3300             }
3301             break
3302         case 'armlb':
3303             if (os != 'Windows_NT') {
3304                 return false
3305             }
3306             // Do not create armlb windows jobs.
3307             return false
3308         case 'arm':
3309             if (os != "Ubuntu" && os != "Windows_NT") {
3310                 return false
3311             }
3312             break
3313         case 'x86':
3314             if (os != "Ubuntu") {
3315                 return false
3316             }
3317             break
3318         case 'x64':
3319             if (!(os in Constants.crossList)) {
3320                 return false
3321             }
3322             if (os == "Windows_NT") {
3323                 return false
3324             }
3325             break
3326         case 'armem':
3327         case 'x86_arm_altjit':
3328         case 'x64_arm64_altjit':
3329             // No flow jobs
3330             return false
3331         default:
3332             println("Unknown architecture: ${architecture}")
3333             assert false
3334             break
3335     }
3336
3337     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3338
3339     // Filter based on scenario in OS.
3340
3341     if (os == 'Windows_NT') {
3342         if (!isArmWindowsScenario(scenario)) {
3343             return false
3344         }
3345     }
3346     else {
3347         // Non-Windows
3348         if (architecture == 'arm64') {
3349             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3350                 return false
3351             }
3352         }
3353         else if (architecture == 'arm') {
3354             if (!(scenario in Constants.validLinuxArmScenarios)) {
3355                 return false
3356             }
3357         }
3358         else if (architecture == 'x86') {
3359             // Linux/x86 only want innerloop and default test
3360             if (!isNormalOrInnerloop) {
3361                 return false
3362             }
3363         }
3364     }
3365
3366     // For CentOS, we only want Checked/Release builds.
3367     if (os == 'CentOS7.1') {
3368         if (configuration != 'Checked' && configuration != 'Release') {
3369             return false
3370         }
3371         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3372             return false
3373         }
3374     }
3375
3376     // For RedHat and Debian, we only do Release builds.
3377     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3378         if (configuration != 'Release') {
3379             return false
3380         }
3381         if (!isNormalOrInnerloop) {
3382             return false
3383         }
3384     }
3385
3386     // Next, filter based on scenario.
3387
3388     if (isJitStressScenario(scenario)) {
3389         if (configuration != 'Checked') {
3390             return false
3391         }
3392
3393         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3394         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3395             return false
3396         }
3397     }
3398     else if (isR2RBaselineScenario(scenario)) {
3399         if (configuration != 'Checked' && configuration != 'Release') {
3400             return false
3401         }
3402     }
3403     else if (isR2RStressScenario(scenario)) {
3404         if (configuration != 'Checked') {
3405             return false
3406         }
3407     }
3408     else {
3409         // Skip scenarios
3410         switch (scenario) {
3411             case 'ilrt':
3412             case 'longgc':
3413             case 'gcsimulator':
3414                 // Long GC tests take a long time on non-Release builds
3415                 // ilrt is also Release only
3416                 if (configuration != 'Release') {
3417                     return false
3418                 }
3419                 break
3420
3421             case 'jitdiff':
3422                 if (configuration != 'Checked') {
3423                     return false
3424                 }
3425                 break
3426
3427             case 'gc_reliability_framework':
3428             case 'standalone_gc':
3429                 if (configuration != 'Release' && configuration != 'Checked') {
3430                     return false
3431                 }
3432                 break
3433
3434             case 'formatting':
3435                 return false
3436             case 'illink':
3437                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3438                     return false
3439                 }
3440                 break
3441
3442             case 'normal':
3443                 // Nothing skipped
3444                 break
3445
3446             case 'innerloop':
3447                 // Nothing skipped
3448                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3449                     return false
3450                 }
3451                 break
3452
3453             default:
3454                 println("Unknown scenario: ${scenario}")
3455                 assert false
3456                 break
3457         }
3458     }
3459
3460     // The job was not filtered out, so we should generate it!
3461     return true
3462 }
3463
3464 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3465 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3466 Constants.allScenarios.each { scenario ->
3467     [true, false].each { isPR ->
3468         Constants.architectureList.each { architecture ->
3469             Constants.configurationList.each { configuration ->
3470                 Constants.osList.each { os ->
3471
3472                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3473                         return
3474                     }
3475
3476                     // Figure out the job name of the CoreCLR build the test will depend on.
3477
3478                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3479                     def inputCoreCLRBuildIsBuildOnly = false
3480                     if (isCoreFxScenario(scenario)) {
3481                         // Every CoreFx test depends on its own unique build.
3482                         inputCoreCLRBuildScenario = scenario
3483                         inputCoreCLRBuildIsBuildOnly = true
3484                     }
3485                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3486                     def inputCoreCLRBuildName = projectFolder + '/' +
3487                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3488
3489                     // Figure out the name of the build job that the test job will depend on.
3490                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3491                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3492
3493                     def inputTestsBuildName = null
3494
3495                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3496                     if (!windowsArmJob) {
3497                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3498
3499                         def inputTestsBuildArch = architecture
3500                         if (architecture == "arm64") {
3501                             // Use the x64 test build for arm64 unix
3502                             inputTestsBuildArch = "x64"
3503                         }
3504                         else if (architecture == "arm") {
3505                             // Use the x86 test build for arm unix
3506                             inputTestsBuildArch = "x86"
3507                         }
3508
3509                         def inputTestsBuildIsBuildOnly = true
3510
3511                         inputTestsBuildName = projectFolder + '/' +
3512                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3513                     }
3514
3515                     // =============================================================================================
3516                     // Create the test job
3517                     // =============================================================================================
3518
3519                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3520
3521                     // =============================================================================================
3522                     // Create a build flow to join together the build and tests required to run this test.
3523                     // =============================================================================================
3524
3525                     def fullTestJobName = projectFolder + '/' + testJob.name
3526                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3527
3528                 } // os
3529             } // configuration
3530         } // architecture
3531     } // isPR
3532 } // scenario
3533
3534 JobReport.Report.generateJobReport(out)
3535
3536 // Make the call to generate the help job
3537 Utilities.createHelperJob(this, project, branch,
3538     "Welcome to the ${project} Repository",  // This is prepended to the help message
3539     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3540
3541 Utilities.addCROSSCheck(this, project, branch)