netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213     ]
 214
 215     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 216     // to Tests.lst file categories to exclude.
 217     //
 218     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 219     // and with the same values, with some commented out, for easier maintenance.
 220     //
 221     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 222     //
 223     def static validArmWindowsScenarios = [
 224                'innerloop':                              [],
 225                'normal':                                 [],
 226                // 'ilrt'
 227                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 228                // 'longgc'
 229                // 'formatting'
 230                // 'gcsimulator'
 231                // 'jitdiff'
 232                // 'standalone_gc'
 233                // 'gc_reliability_framework'
 234                // 'illink'
 235                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 246                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 248                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 249                'tieredcompilation':                      [],
 250                'forcerelocs':                            [],
 251                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 270                // 'jitsse2only'                          // Only relevant to xarch
 271                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 272                // 'jitincompletehwintrinsic'
 273                // 'jitx86hwintrinsicnoavx'
 274                // 'jitx86hwintrinsicnoavx2'
 275                // 'jitx86hwintrinsicnosimd'
 276                // 'jitnox86hwintrinsic'
 277                'corefx_baseline':                        [],    // corefx tests don't use smarty
 278                'corefx_minopts':                         [],    // corefx tests don't use smarty
 279                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 280                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 281                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 283                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 284                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 285                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 286                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 290                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 291                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 292                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 293                'heapverify1':                            [],
 294                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 295                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 297                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 300
 301                //
 302                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 303                //
 304
 305                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 306                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 324     ]
 325
 326     def static validLinuxArm64Scenarios = [
 327                'innerloop',
 328                'normal',
 329                'r2r',
 330                'gcstress0x3',
 331                'gcstress0xc'
 332     ]
 333
 334     def static validLinuxArmScenarios = [
 335                'innerloop',
 336                'normal',
 337                'r2r',
 338                'r2r_jitstress1',
 339                'r2r_jitstress2',
 340                'r2r_jitstressregs1',
 341                'r2r_jitstressregs2',
 342                'r2r_jitstressregs3',
 343                'r2r_jitstressregs4',
 344                'r2r_jitstressregs8',
 345                'r2r_jitstressregs0x10',
 346                'r2r_jitstressregs0x80',
 347                'r2r_jitstressregs0x1000',
 348                'r2r_jitminopts',
 349                'r2r_jitforcerelocs',
 350                'r2r_gcstress15',
 351                'minopts',
 352                'forcerelocs',
 353                'jitstress1',
 354                'jitstress2',
 355                'jitstressregs1',
 356                'jitstressregs2',
 357                'jitstressregs3',
 358                'jitstressregs4',
 359                'jitstressregs8',
 360                'jitstressregs0x10',
 361                'jitstressregs0x80',
 362                'jitstressregs0x1000',
 363                'jitstress2_jitstressregs1',
 364                'jitstress2_jitstressregs2',
 365                'jitstress2_jitstressregs3',
 366                'jitstress2_jitstressregs4',
 367                'jitstress2_jitstressregs8',
 368                'jitstress2_jitstressregs0x10',
 369                'jitstress2_jitstressregs0x80',
 370                'jitstress2_jitstressregs0x1000',
 371                'tailcallstress',
 372                'gcstress0x3',
 373                'gcstress0xc',
 374                'zapdisable',
 375                'heapverify1',
 376                'gcstress0xc_zapdisable',
 377                'gcstress0xc_zapdisable_jitstress2',
 378                'gcstress0xc_zapdisable_heapverify1',
 379                'gcstress0xc_jitstress1',
 380                'gcstress0xc_jitstress2',
 381                'gcstress0xc_minopts_heapverify1'
 382     ]
 383
 384     def static configurationList = ['Debug', 'Checked', 'Release']
 385
 386     // This is the set of architectures
 387     // Some of these are pseudo-architectures:
 388     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 389     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 390     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 391     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 392     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 393
 394     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 395     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 396 }
 397
 398 // **************************************************************
 399 // Create some specific views
 400 //
 401 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 402 // views based on a single regular expression. These views will be generated by adding a
 403 // specific set of jobs to them.
 404 //
 405 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 406 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 407 // those are required, don't add them (which simplifies the view pages, as well).
 408 // **************************************************************
 409
 410 class Views {
 411     def static MergeJobView = null
 412     def static PeriodicJobView = null
 413     def static ArchitectureViews = [:]
 414     def static OSViews = [:]
 415 }
 416
 417 // MergeJobView: include all jobs that execute when a PR change is merged.
 418 Views.MergeJobView = listView('Merge') {
 419     recurse()
 420     columns {
 421         status()
 422         weather()
 423         name()
 424         lastSuccess()
 425         lastFailure()
 426         lastDuration()
 427         buildButton()
 428     }
 429 }
 430
 431 // PeriodicJobView: include all jobs that execute on a schedule
 432 Views.PeriodicJobView = listView('Periodic') {
 433     recurse()
 434     columns {
 435         status()
 436         weather()
 437         name()
 438         lastSuccess()
 439         lastFailure()
 440         lastDuration()
 441         buildButton()
 442     }
 443 }
 444
 445 // Create a view for non-PR jobs for each architecture.
 446 Constants.architectureList.each { architecture ->
 447     Views.ArchitectureViews[architecture] = listView(architecture) {
 448         recurse()
 449         columns {
 450             status()
 451             weather()
 452             name()
 453             lastSuccess()
 454             lastFailure()
 455             lastDuration()
 456             buildButton()
 457         }
 458     }
 459 }
 460
 461 // Create a view for non-PR jobs for each OS.
 462 Constants.osList.each { os ->
 463     // Don't create one for the special 'Windows_NT_BuildOnly'
 464     if (os == 'Windows_NT_BuildOnly') {
 465         return
 466     }
 467     Views.OSViews[os] = listView(os) {
 468         recurse()
 469         columns {
 470             status()
 471             weather()
 472             name()
 473             lastSuccess()
 474             lastFailure()
 475             lastDuration()
 476             buildButton()
 477         }
 478     }
 479 }
 480
 481 def static addToMergeView(def job) {
 482     Views.MergeJobView.with {
 483         jobs {
 484             name(job.name)
 485         }
 486     }
 487 }
 488
 489 def static addToPeriodicView(def job) {
 490     Views.PeriodicJobView.with {
 491         jobs {
 492             name(job.name)
 493         }
 494     }
 495 }
 496
 497 def static addToViews(def job, def isPR, def architecture, def os) {
 498     if (isPR) {
 499         // No views want PR jobs currently.
 500         return
 501     }
 502
 503     // Add to architecture view.
 504     Views.ArchitectureViews[architecture].with {
 505         jobs {
 506             name(job.name)
 507         }
 508     }
 509
 510     // Add to OS view.
 511     Views.OSViews[os].with {
 512         jobs {
 513             name(job.name)
 514         }
 515     }
 516 }
 517
 518 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 519     addToPeriodicView(job)
 520     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 521 }
 522
 523 def static addGithubPushTriggerHelper(def job) {
 524     addToMergeView(job)
 525     Utilities.addGithubPushTrigger(job)
 526 }
 527
 528
 529 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 530     assert os instanceof String
 531     assert architecture instanceof String
 532
 533     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 534     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 535
 536     if (!(architecture in armArches)) {
 537         assert options == null
 538         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 539
 540         return
 541     }
 542
 543     // This is an arm(64) job.
 544     //
 545     // There are several options.
 546     //
 547     // Windows_NT
 548     //
 549     // Arm32 (Build) -> latest-arm64
 550     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 551     // Arm32 (Test)  -> arm64-windows_nt
 552     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 553     //
 554     // Arm64 (Build) -> latest-arm64
 555     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 556     // Arm64 (Test)  -> arm64-windows_nt
 557     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 558     //
 559     // Ubuntu
 560     //
 561     // Arm32 emulator (Build, Test) -> arm-cross-latest
 562     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 563     //
 564     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 565     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 566     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 567     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 568     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 569     //       |-> os == "Ubuntu" && (architecture == "arm")
 570     //
 571     // Arm64 (Build) -> arm64-cross-latest
 572     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 573     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 574     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 575     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 576     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 577
 578     // This has to be a arm arch
 579     assert architecture in armArches
 580     if (os == "Windows_NT") {
 581         // arm32/arm64 Windows jobs share the same machines for now
 582         def isBuild = options['use_arm64_build_machine'] == true
 583
 584         if (isBuild == true) {
 585             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 586         } else {
 587             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 588         }
 589     } else {
 590         assert os != 'Windows_NT'
 591         assert os in supportedArmLinuxOs
 592
 593         if (architecture == 'arm64') {
 594             if ((options != null) && (options['is_build_only'] == true)) {
 595                 // Arm64 Linux build machine
 596                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 597             } else {
 598                 // Arm64 Linux test machines
 599                 if ((options != null) && (options['large_pages'] == true)) {
 600                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 601                 } else {
 602                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 603                 }
 604             }
 605         }
 606         else if (architecture == 'armem') {
 607             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 608             // using Docker.
 609             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 610         }
 611         else {
 612             // arm Ubuntu on hardware.
 613             assert (architecture == 'arm') && (os == 'Ubuntu')
 614             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 615             def isBuild = (options != null) && (options['is_build_job'] == true)
 616             if (isFlow || isBuild) {
 617                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 618                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 619                 // arm hardware.
 620                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 621             } else {
 622                 // arm Ubuntu test machine
 623                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 624                 // Utilities.setMachineAffinity. Just add the machine affinity
 625                 // manually. We specify the Helix queue name here.
 626                 job.with {
 627                     label('ubuntu.1404.arm32.open')
 628                 }
 629             }
 630         }
 631     }
 632 }
 633
 634 // setJobMachineAffinity: compute the machine affinity options for a job,
 635 // then set the job with those affinity options.
 636 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 637 {
 638     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 639            (!isBuildJob && isTestJob  && !isFlowJob) ||
 640            (!isBuildJob && !isTestJob && isFlowJob)
 641
 642     def affinityOptions = null
 643     def affinityArchitecture = architecture
 644
 645     if (os == "Windows_NT") {
 646         if (architecture in Constants.armWindowsCrossArchitectureList) {
 647             if (isBuildJob) {
 648                 affinityOptions = [ "use_arm64_build_machine" : true ]
 649             } else if (isTestJob) {
 650                 affinityOptions = [ "use_arm64_build_machine" : false ]
 651             } else if (isFlowJob) {
 652                 // For the flow jobs set the machine affinity as x64
 653                 affinityArchitecture = 'x64'
 654             }
 655         }
 656     }
 657     else {
 658         if (architecture == 'arm64') {
 659             if (isBuildJob) {
 660                 affinityOptions = ['is_build_only': true]
 661             } else if (isTestJob) {
 662                 affinityOptions = [ "large_pages" : false ]
 663             }
 664         }
 665         else if (architecture == 'arm') {
 666             if (isBuildJob) {
 667                 affinityOptions = ['is_build_job': true]
 668             } else if (isFlowJob) {
 669                 affinityOptions = ['is_flow_job': true]
 670             }
 671         }
 672     }
 673
 674     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 675 }
 676
 677 def static isGCStressRelatedTesting(def scenario) {
 678     // The 'r2r_gcstress15' scenario is a basic scenario.
 679     // Detect it and make it a GCStress related.
 680     if (scenario == 'r2r_gcstress15')
 681     {
 682         return true;
 683     }
 684
 685     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 686     def scenarioName = scenario.toLowerCase()
 687     def isGCStressTesting = false
 688     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 689         if (k in gcStressTestEnvVars) {
 690             isGCStressTesting = true;
 691         }
 692     }
 693     return isGCStressTesting
 694 }
 695
 696 def static isCoreFxScenario(def scenario) {
 697     def corefx_prefix = 'corefx_'
 698     if (scenario.length() < corefx_prefix.length()) {
 699         return false
 700     }
 701     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 702 }
 703
 704 def static isR2RBaselineScenario(def scenario) {
 705     return (scenario == 'r2r')
 706 }
 707
 708 def static isR2RStressScenario(def scenario) {
 709     return Constants.r2rStressScenarios.containsKey(scenario)
 710 }
 711
 712 def static isR2RScenario(def scenario) {
 713     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 714 }
 715
 716 def static isJitStressScenario(def scenario) {
 717     return Constants.jitStressModeScenarios.containsKey(scenario)
 718 }
 719
 720 def static isLongGc(def scenario) {
 721     return (scenario == 'longgc' || scenario == 'gcsimulator')
 722 }
 723
 724 def static isJitDiff(def scenario) {
 725     return (scenario == 'jitdiff')
 726 }
 727
 728 def static isGcReliabilityFramework(def scenario) {
 729     return (scenario == 'gc_reliability_framework')
 730 }
 731
 732 def static isArmWindowsScenario(def scenario) {
 733     return Constants.validArmWindowsScenarios.containsKey(scenario)
 734 }
 735
 736 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 737     if (isBuildOnly == true) {
 738         os = 'Windows_NT_BuildOnly'
 739     }
 740
 741     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 742
 743     if (validOsPrTriggerArchConfigs == null) {
 744         return false
 745     }
 746
 747     if (validOsPrTriggerArchConfigs[architecture] != null) {
 748         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 749
 750         if (!(configuration in validOsPrTriggerConfigs)) {
 751             return false
 752         }
 753     } else {
 754         return false
 755     }
 756
 757     return true
 758 }
 759
 760 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 761     // 2 hours (120 minutes) is the default timeout
 762     def timeout = 120
 763     def innerLoop = (scenario == "innerloop")
 764
 765     if (!innerLoop) {
 766         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 767         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 768         timeout = 240
 769     }
 770
 771     if (!isBuildOnly) {
 772         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 773         if (isGCStressRelatedTesting(scenario)) {
 774             timeout = 4320
 775         }
 776         else if (isCoreFxScenario(scenario)) {
 777             timeout = 360
 778         }
 779         else if (isJitStressScenario(scenario)) {
 780             timeout = 300
 781         }
 782         else if (isR2RBaselineScenario(scenario)) {
 783             timeout = 240
 784         }
 785         else if (isLongGc(scenario)) {
 786             timeout = 1440
 787         }
 788         else if (isJitDiff(scenario)) {
 789             timeout = 240
 790         }
 791         else if (isGcReliabilityFramework(scenario)) {
 792             timeout = 1440
 793         }
 794         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 795             timeout = 240
 796         }
 797
 798         if (architecture == 'arm') {
 799             // ARM32 machines are particularly slow.
 800             timeout += 120
 801         }
 802     }
 803
 804     if (configuration == 'Debug') {
 805         // Debug runs can be very slow. Add an hour.
 806         timeout += 60
 807     }
 808
 809     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 810         // AltJit runs compile all methods twice.
 811         timeout *= 2
 812     }
 813
 814     // If we've changed the timeout from the default, set it in the job.
 815
 816     if (timeout != 120) {
 817         Utilities.setJobTimeout(newJob, timeout)
 818     }
 819 }
 820
 821 def static getJobFolder(def scenario) {
 822     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 823         return 'jitstress'
 824     }
 825     if (scenario == 'illink') {
 826         return 'illink'
 827     }
 828     return ''
 829 }
 830
 831 def static getStressModeDisplayName(def scenario) {
 832     def displayStr = ''
 833     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 834         def prefixLength = 'COMPlus_'.length()
 835         if (k.length() >= prefixLength) {
 836             def modeName = k.substring(prefixLength, k.length())
 837             displayStr += ' ' + modeName + '=' + v
 838         }
 839     }
 840
 841     if (isCoreFxScenario(scenario)) {
 842         displayStr = ('CoreFx ' + displayStr).trim()
 843     }
 844
 845     return displayStr
 846 }
 847
 848 def static getR2RDisplayName(def scenario) {
 849     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 850     def displayStr = scenario
 851     def prefixLength = 'r2r_'.length()
 852     if (displayStr.length() >= prefixLength) {
 853         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 854     } else if (scenario == 'r2r') {
 855         displayStr = "R2R"
 856     }
 857     return displayStr
 858 }
 859
 860 //
 861 // Functions to create an environment script.
 862 //      envScriptCreate -- initialize the script (call first)
 863 //      envScriptFinalize -- finalize the script (call last)
 864 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 865 //      envScriptAppendExistingScript -- append an existing script to the generated script
 866 //
 867 // Each script returns a string of commands. Concatenate all the strings together before
 868 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 869 //
 870
 871 // Initialize the environment setting script.
 872 def static envScriptCreate(def os, def stepScriptLocation) {
 873     def stepScript = ''
 874     if (os == 'Windows_NT') {
 875         stepScript += "echo Creating TestEnv script\r\n"
 876         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 877
 878         // Create at least an empty script.
 879         stepScript += "echo. > ${stepScriptLocation}\r\n"
 880     }
 881     else {
 882         stepScript += "echo Creating environment setting script\n"
 883         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 884     }
 885
 886     return stepScript
 887 }
 888
 889 // Generates the string for setting stress mode variables.
 890 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 891     def stepScript = ''
 892     if (os == 'Windows_NT') {
 893         stressModeVars.each{ k, v ->
 894             // Write out what we are writing to the script file
 895             stepScript += "echo Setting ${k}=${v}\r\n"
 896             // Write out the set itself to the script file`
 897             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 898         }
 899     }
 900     else {
 901         stressModeVars.each{ k, v ->
 902             // Write out what we are writing to the script file
 903             stepScript += "echo Setting ${k}=${v}\n"
 904             // Write out the set itself to the script file`
 905             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 906         }
 907     }
 908
 909     return stepScript
 910 }
 911
 912 // Append an existing script to an environment script.
 913 // Returns string of commands to do this.
 914 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 915     assert (os == 'Windows_NT')
 916     def stepScript = ''
 917
 918     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 919     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 920
 921     return stepScript
 922 }
 923
 924 // Finalize an environment setting script.
 925 // Returns string of commands to do this.
 926 def static envScriptFinalize(def os, def stepScriptLocation) {
 927     def stepScript = ''
 928
 929     if (os == 'Windows_NT') {
 930         // Display the resulting script. This is useful when looking at the output log file.
 931         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 932         stepScript += "type ${stepScriptLocation}\r\n"
 933     }
 934     else {
 935         stepScript += "chmod +x ${stepScriptLocation}\n"
 936     }
 937
 938     return stepScript
 939 }
 940
 941 def static isNeedDocker(def architecture, def os, def isBuild) {
 942     if (isBuild) {
 943         if (architecture == 'x86' && os == 'Ubuntu') {
 944             return true
 945         }
 946         else if (architecture == 'armem') {
 947             return true
 948         }
 949         else if (architecture == 'arm') {
 950             if (os == 'Ubuntu') {
 951                 return true
 952             }
 953         }
 954     }
 955     else {
 956         if (architecture == 'x86' && os == 'Ubuntu') {
 957             return true
 958         }
 959     }
 960     return false
 961 }
 962
 963 def static getDockerImageName(def architecture, def os, def isBuild) {
 964     // We must change some docker private images to official later
 965     if (isBuild) {
 966         if (architecture == 'x86' && os == 'Ubuntu') {
 967             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 968         }
 969         else if (architecture == 'armem') {
 970             if (os == 'Ubuntu') {
 971                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180405193556"
 972             }
 973             else if (os == 'Ubuntu16.04') {
 974                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180404203310"
 975             }
 976             else if (os == 'Tizen') {
 977                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 978             }
 979         }
 980         else if (architecture == 'arm') {
 981             if (os == 'Ubuntu') {
 982                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180405193556"
 983             }
 984         }
 985     }
 986     else {
 987         if (architecture == 'x86' && os == 'Ubuntu') {
 988             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 989         }
 990     }
 991     println("Unknown architecture to use docker: ${architecture} ${os}");
 992     assert false
 993 }
 994
 995
 996 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
 997 def static jobRequiresLimitedHardware(def architecture, def os) {
 998     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
 999         // These test jobs require ARM64 hardware
1000         return true
1001     }
1002     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1003         // These test jobs require Linux/arm32 hardware
1004         return true
1005     }
1006     else {
1007         return false
1008     }
1009 }
1010
1011 // Calculates the name of the build job based on some typical parameters.
1012 //
1013 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1014     // If the architecture is x64, do not add that info into the build name.
1015     // Need to change around some systems and other builds to pick up the right builds
1016     // to do that.
1017
1018     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1019     if (isBuildOnly) {
1020         suffix += '_bld'
1021     }
1022     def baseName = ''
1023     switch (architecture) {
1024         case 'x64':
1025             if (scenario == 'normal') {
1026                 // For now we leave x64 off of the name for compatibility with other jobs
1027                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1028             }
1029             else if (scenario == 'formatting') {
1030                 // we don't care about the configuration for the formatting job. It runs all configs
1031                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1032             }
1033             else {
1034                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1035             }
1036             break
1037         case 'arm64':
1038             if (os.toLowerCase() == "windows_nt") {
1039                 // These are cross builds
1040                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1041             }
1042             else {
1043                 // Defaults to a small page size set of machines.
1044                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1045             }
1046             break
1047         case 'armem':
1048             // These are cross builds
1049             if (os == 'Tizen') {
1050                 // ABI: softfp
1051                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1052             }
1053             else {
1054                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1055             }
1056             break
1057         case 'armlb':
1058         case 'arm':
1059             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1060             break
1061         case 'x86':
1062         case 'x86_arm_altjit':
1063         case 'x64_arm64_altjit':
1064             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1065             break
1066         default:
1067             println("Unknown architecture: ${architecture}");
1068             assert false
1069             break
1070     }
1071
1072     return baseName + suffix
1073 }
1074
1075 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1076
1077     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1078     if (os == 'Windows_NT') {
1079         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1080             if (!(branch in Constants.WindowsArm64Branches)) {
1081                 return
1082             }
1083         }
1084     }
1085
1086     if ((architecture == 'arm') && (os != 'Windows_NT') && isGCStressRelatedTesting(scenario)) {
1087         // Non-Windows Arm GCStress jobs currently don't get cron or push triggers (until they are functional).
1088         // See https://github.com/dotnet/coreclr/issues/17241.
1089         return
1090     }
1091
1092     // Check scenario.
1093     switch (scenario) {
1094         case 'innerloop':
1095             break
1096         case 'normal':
1097             switch (architecture) {
1098                 case 'x64':
1099                 case 'x86':
1100                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1101                         addPeriodicTriggerHelper(job, '@daily')
1102                     }
1103                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     break
1107                 case 'arm':
1108                     if (os == 'Windows_NT') {
1109                         addGithubPushTriggerHelper(job)
1110                     }
1111                     else {
1112                         // Currently no push triggers, with limited arm Linux hardware.
1113                         // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1114                         assert os == 'Ubuntu'
1115                         if (isFlowJob) {
1116                             addPeriodicTriggerHelper(job, '@daily')
1117                         }
1118                     }
1119                     break
1120                 case 'armem':
1121                 case 'armlb':
1122                 case 'x86_arm_altjit':
1123                 case 'x64_arm64_altjit':
1124                     addGithubPushTriggerHelper(job)
1125                     break
1126                 case 'arm64':
1127                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1128                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1129                     break
1130                 default:
1131                     println("Unknown architecture: ${architecture}");
1132                     assert false
1133                     break
1134             }
1135             break
1136         case 'r2r':
1137             assert !(os in bidailyCrossList)
1138             // r2r gets a push trigger for checked/release
1139             if (configuration == 'Checked' || configuration == 'Release') {
1140                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1141                 if (architecture == 'x64' && os != 'OSX10.12') {
1142                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1143                     if (isFlowJob || os == 'Windows_NT') {
1144                         addGithubPushTriggerHelper(job)
1145                     }
1146                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1147                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1148                     if (isFlowJob) {
1149                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1150                     }
1151                 }
1152                 // For x86, only add per-commit jobs for Windows
1153                 else if (architecture == 'x86') {
1154                     if (os == 'Windows_NT') {
1155                         addGithubPushTriggerHelper(job)
1156                     }
1157                 }
1158                 // arm64 r2r jobs should only run daily.
1159                 else if (architecture == 'arm64') {
1160                     if (os == 'Windows_NT') {
1161                         addPeriodicTriggerHelper(job, '@daily')
1162                     }
1163                 }
1164             }
1165             break
1166         case 'r2r_jitstress1':
1167         case 'r2r_jitstress2':
1168         case 'r2r_jitstressregs1':
1169         case 'r2r_jitstressregs2':
1170         case 'r2r_jitstressregs3':
1171         case 'r2r_jitstressregs4':
1172         case 'r2r_jitstressregs8':
1173         case 'r2r_jitstressregs0x10':
1174         case 'r2r_jitstressregs0x80':
1175         case 'r2r_jitstressregs0x1000':
1176         case 'r2r_jitminopts':
1177         case 'r2r_jitforcerelocs':
1178         case 'r2r_gcstress15':
1179             assert !(os in bidailyCrossList)
1180
1181             // GCStress=C is currently not supported on OS X
1182             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1183                 break
1184             }
1185
1186             // GC Stress 15 r2r gets a push trigger for checked/release
1187             if (configuration == 'Checked' || configuration == 'Release') {
1188                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1189                 if (architecture == 'x64') {
1190                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1191                     if (isFlowJob || os == 'Windows_NT') {
1192                         // Add a weekly periodic trigger
1193                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1194                     }
1195                 }
1196                 // For x86, only add per-commit jobs for Windows
1197                 else if (architecture == 'x86') {
1198                     if (os == 'Windows_NT') {
1199                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1200                     }
1201                 }
1202             }
1203             break
1204         case 'longgc':
1205             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1206             assert configuration == 'Release'
1207             assert architecture == 'x64'
1208             addPeriodicTriggerHelper(job, '@daily')
1209             // TODO: Add once external email sending is available again
1210             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1211             break
1212         case 'gcsimulator':
1213             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1214             assert configuration == 'Release'
1215             assert architecture == 'x64'
1216             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1217             // TODO: Add once external email sending is available again
1218             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1219             break
1220         case 'standalone_gc':
1221             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1222             assert (configuration == 'Release' || configuration == 'Checked')
1223             // TODO: Add once external email sending is available again
1224             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1225             addPeriodicTriggerHelper(job, '@daily')
1226             break
1227         case 'gc_reliability_framework':
1228             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1229             assert (configuration == 'Release' || configuration == 'Checked')
1230             // Only triggered by phrase.
1231             break
1232         case 'ilrt':
1233             assert !(os in bidailyCrossList)
1234             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1235             if (architecture == 'x64' && configuration == 'Release') {
1236                 // We don't expect to see a job generated except in these scenarios
1237                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1238                 if (isFlowJob || os == 'Windows_NT') {
1239                     addPeriodicTriggerHelper(job, '@daily')
1240                 }
1241             }
1242             break
1243         case 'jitdiff':
1244             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1245             assert configuration == 'Checked'
1246             assert (architecture == 'x64' || architecture == 'x86')
1247             addGithubPushTriggerHelper(job)
1248             break
1249         case 'formatting':
1250             assert (os == 'Windows_NT' || os == "Ubuntu")
1251             assert architecture == 'x64'
1252             addGithubPushTriggerHelper(job)
1253             break
1254         case 'jitstressregs1':
1255         case 'jitstressregs2':
1256         case 'jitstressregs3':
1257         case 'jitstressregs4':
1258         case 'jitstressregs8':
1259         case 'jitstressregs0x10':
1260         case 'jitstressregs0x80':
1261         case 'jitstressregs0x1000':
1262         case 'minopts':
1263         case 'forcerelocs':
1264         case 'jitstress1':
1265         case 'jitstress2':
1266         case 'jitstress2_jitstressregs1':
1267         case 'jitstress2_jitstressregs2':
1268         case 'jitstress2_jitstressregs3':
1269         case 'jitstress2_jitstressregs4':
1270         case 'jitstress2_jitstressregs8':
1271         case 'jitstress2_jitstressregs0x10':
1272         case 'jitstress2_jitstressregs0x80':
1273         case 'jitstress2_jitstressregs0x1000':
1274         case 'tailcallstress':
1275         case 'jitsse2only':
1276         case 'jitnosimd':
1277         case 'jitnox86hwintrinsic':
1278         case 'jitincompletehwintrinsic':
1279         case 'jitx86hwintrinsicnoavx':
1280         case 'jitx86hwintrinsicnoavx2':
1281         case 'jitx86hwintrinsicnosimd':
1282         case 'corefx_baseline':
1283         case 'corefx_minopts':
1284         case 'corefx_jitstress1':
1285         case 'corefx_jitstress2':
1286         case 'corefx_jitstressregs1':
1287         case 'corefx_jitstressregs2':
1288         case 'corefx_jitstressregs3':
1289         case 'corefx_jitstressregs4':
1290         case 'corefx_jitstressregs8':
1291         case 'corefx_jitstressregs0x10':
1292         case 'corefx_jitstressregs0x80':
1293         case 'corefx_jitstressregs0x1000':
1294         case 'zapdisable':
1295             if (os == 'CentOS7.1') {
1296                 break
1297             }
1298             if (os in bidailyCrossList) {
1299                 break
1300             }
1301             assert (os == 'Windows_NT') || (os in Constants.crossList)
1302             if (jobRequiresLimitedHardware(architecture, os)) {
1303                 addPeriodicTriggerHelper(job, '@weekly')
1304             }
1305             else {
1306                 addPeriodicTriggerHelper(job, '@daily')
1307             }
1308             break
1309         case 'heapverify1':
1310         case 'gcstress0x3':
1311             if (os == 'CentOS7.1') {
1312                 break
1313             }
1314             if (os in bidailyCrossList) {
1315                 break
1316             }
1317             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1318                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1319                 break
1320             }
1321             assert (os == 'Windows_NT') || (os in Constants.crossList)
1322             addPeriodicTriggerHelper(job, '@weekly')
1323             break
1324         case 'gcstress0xc':
1325         case 'gcstress0xc_zapdisable':
1326         case 'gcstress0xc_zapdisable_jitstress2':
1327         case 'gcstress0xc_zapdisable_heapverify1':
1328         case 'gcstress0xc_jitstress1':
1329         case 'gcstress0xc_jitstress2':
1330         case 'gcstress0xc_minopts_heapverify1':
1331             if (os == 'CentOS7.1') {
1332                 break
1333             }
1334             if (os == 'OSX10.12') {
1335                 // GCStress=C is currently not supported on OS X
1336                 break
1337             }
1338             if (os in bidailyCrossList) {
1339                 break
1340             }
1341             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1342                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1343                 break
1344             }
1345             assert (os == 'Windows_NT') || (os in Constants.crossList)
1346             addPeriodicTriggerHelper(job, '@weekly')
1347             break
1348
1349         case 'illink':
1350             // Testing on other operating systems TBD
1351             assert (os == 'Windows_NT' || os == 'Ubuntu')
1352             if (architecture == 'x64' || architecture == 'x86') {
1353                 if (configuration == 'Checked') {
1354                     addPeriodicTriggerHelper(job, '@daily')
1355                 }
1356             }
1357             break
1358
1359         case 'tieredcompilation':
1360         case 'corefx_tieredcompilation':
1361             // No periodic jobs just yet, still testing
1362             break
1363
1364         default:
1365             println("Unknown scenario: ${scenario}");
1366             assert false
1367             break
1368     }
1369     return
1370 }
1371
1372 // **************************
1373 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1374 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1375 // builds will do a couple extra steps.
1376 // **************************
1377
1378 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1379 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1380 // it's just used for internal builds
1381 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1382 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1383     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1384
1385     if (isWindowsBuildOnlyJob) {
1386         return
1387     }
1388
1389     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1390     // Non pull request builds.
1391     if (!isPR) {
1392         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1393         return
1394     }
1395
1396      def arm64Users = [
1397         'AndyAyersMS',
1398         'briansull',
1399         'BruceForstall',
1400         'CarolEidt',
1401         'cmckinsey',
1402         'echesakovMSFT',
1403         'erozenfeld',
1404         'janvorli',
1405         'jashook',
1406         'JosephTremoulet',
1407         'pgodeq',
1408         'russellhadley',
1409         'RussKeldorph',
1410         'sandreenko',
1411         'sdmaclea',
1412         'swaroop-sridhar',
1413         'jkotas',
1414         'markwilkie',
1415         'weshaggard'
1416     ]
1417
1418     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1419     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1420     def osGroup = getOSGroup(os)
1421     switch (architecture) {
1422         case 'x64': // editor brace matching: {
1423             if (scenario == 'formatting') {
1424                 assert configuration == 'Checked'
1425                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1426                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1427                 }
1428
1429                 break
1430             }
1431
1432             switch (os) {
1433                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1434                 case 'Debian8.4':
1435                 case 'RHEL7.2':
1436                     if (scenario == 'innerloop') {
1437                         assert !isFlowJob
1438                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1439                     }
1440                     else if (scenario == 'normal') {
1441                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1442                     }
1443                     break
1444
1445                 case 'Ubuntu16.04':
1446                     assert !isFlowJob
1447                     assert scenario != 'innerloop'
1448                     // Distinguish with the other architectures (arm and x86)
1449                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1450                     break
1451
1452                 case 'Fedora24':
1453                 case 'Ubuntu16.10':
1454                     assert !isFlowJob
1455                     assert scenario != 'innerloop'
1456                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1457                     break
1458
1459                 case 'Ubuntu':
1460                     if (scenario == 'illink') {
1461                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1462                         break
1463                     }
1464                     // fall through
1465
1466                 case 'OSX10.12':
1467                     // Triggers on the non-flow jobs aren't necessary here
1468                     // Corefx testing uses non-flow jobs.
1469                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1470                         break
1471                     }
1472                     switch (scenario) {
1473                         case 'innerloop':
1474                             // PR Triggered jobs. These jobs will run pri0 tests.
1475                             if (configuration == 'Checked') {
1476                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1477                             }
1478                             break
1479
1480                         case 'normal':
1481                             // OSX uses checked for default PR tests
1482                             if (configuration == 'Checked') {
1483                                 // Default trigger
1484                                 assert !job.name.contains("centos")
1485                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1486                             }
1487                             break
1488
1489                         case 'jitdiff':
1490                             if (configuration == 'Checked') {
1491                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1492                             }
1493                             break
1494
1495                         case 'ilrt':
1496                             if (configuration == 'Release') {
1497                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1498                             }
1499                             break
1500
1501                         case 'longgc':
1502                             if (configuration == 'Release') {
1503                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1504                             }
1505                             break
1506
1507                         case 'gcsimulator':
1508                             if (configuration == 'Release') {
1509                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1510                             }
1511                             break
1512
1513                         case 'standalone_gc':
1514                             if (configuration == 'Release' || configuration == 'Checked') {
1515                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1516                             }
1517                             break
1518
1519                         case 'gc_reliability_framework':
1520                             if (configuration == 'Release' || configuration == 'Checked') {
1521                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1522                             }
1523                             break
1524
1525                         default:
1526                             if (isJitStressScenario(scenario)) {
1527                                 def displayStr = getStressModeDisplayName(scenario)
1528                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1529                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1530                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1531                             }
1532                             else if (isR2RScenario(scenario)) {
1533                                 if (configuration == 'Release' || configuration == 'Checked') {
1534                                     def displayStr = getR2RDisplayName(scenario)
1535                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1536                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1537                                 }
1538                             }
1539                             else {
1540                                 println("Unknown scenario: ${scenario}");
1541                                 assert false
1542                             }
1543                             break
1544
1545                     }
1546                     break
1547
1548                 case 'CentOS7.1':
1549                     switch (scenario) {
1550                         case 'innerloop':
1551                             // CentOS uses checked for default PR tests while debug is build only
1552                             if (configuration == 'Debug') {
1553                                 // Default trigger
1554                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1555                             }
1556
1557                             // Make sure this is a flow job to get build and test.
1558                             if (configuration == 'Checked' && isFlowJob) {
1559                                 assert job.name.contains("flow")
1560                                 // Default trigger
1561                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1562                             }
1563                             break
1564
1565                         case 'normal':
1566                             // Make sure this is a flow job to get build and test.
1567                             if (configuration == 'Checked' && isFlowJob) {
1568                                 assert job.name.contains("flow")
1569                                 // Default trigger
1570                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1571                             }
1572                             break
1573
1574                         default:
1575                             if (isR2RScenario(scenario)) {
1576                                 if (configuration == 'Release' || configuration == 'Checked') {
1577                                     def displayStr = getR2RDisplayName(scenario)
1578                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1579                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1580                                 }
1581                             }
1582                             break
1583
1584                     }
1585                     break
1586
1587                 case 'Windows_NT':
1588                     switch (scenario) {
1589                         case 'innerloop':
1590                             // Default trigger
1591                             if (configuration == 'Checked' || configuration == 'Release') {
1592                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1593                             }
1594                             break
1595
1596                         case 'normal':
1597                             if (configuration == 'Checked') {
1598                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1599                             }
1600                             break
1601
1602                         case 'jitdiff':
1603                             if (configuration == 'Checked') {
1604                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1605                             }
1606                             break
1607
1608                         case 'ilrt':
1609                             if (configuration == 'Release') {
1610                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1611                             }
1612                             break
1613
1614                         case 'longgc':
1615                             if (configuration == 'Release') {
1616                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1617                             }
1618                             break
1619
1620                         case 'gcsimulator':
1621                             if (configuration == 'Release') {
1622                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1623                             }
1624                             break
1625
1626                         case 'standalone_gc':
1627                             if (configuration == 'Release' || configuration == 'Checked') {
1628                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1629                             }
1630                             break
1631
1632                         case 'gc_reliability_framework':
1633                             if (configuration == 'Release' || configuration == 'Checked') {
1634                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1635                             }
1636                             break
1637
1638                         case 'illink':
1639                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1640                             break
1641
1642                         default:
1643                             if (isJitStressScenario(scenario)) {
1644                                 def displayStr = getStressModeDisplayName(scenario)
1645                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1646                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1647                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1648                             }
1649                             else if (isR2RScenario(scenario)) {
1650                                 if (configuration == 'Release' || configuration == 'Checked') {
1651                                     def displayStr = getR2RDisplayName(scenario)
1652                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1653                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1654                                 }
1655                             }
1656                             else {
1657                                 println("Unknown scenario: ${scenario}");
1658                                 assert false
1659                             }
1660                             break
1661
1662                     }
1663                     break
1664
1665                 default:
1666                     println("Unknown os: ${os}");
1667                     assert false
1668                     break
1669
1670             }
1671
1672             break
1673         // editor brace matching: }
1674
1675         case 'armem': // editor brace matching: {
1676             job.with {
1677                 publishers {
1678                     azureVMAgentPostBuildAction {
1679                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1680                     }
1681                 }
1682             }
1683
1684             switch (os) {
1685                 case 'Ubuntu':
1686                 case 'Ubuntu16.04':
1687                     assert scenario != 'innerloop'
1688                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1689                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1690                     break
1691
1692                 case 'Tizen':
1693                     architecture = 'armel'
1694
1695                     assert scenario != 'innerloop'
1696                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1697                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1698                     break
1699             }
1700
1701             break
1702         // editor brace matching: }
1703
1704         case 'armlb':
1705         case 'arm': // editor brace matching: {
1706
1707             // Triggers on the non-flow jobs aren't necessary
1708             if (!isFlowJob) {
1709                 break
1710             }
1711
1712             // Set up a private trigger
1713             def contextString = "${os} ${architecture} Cross ${configuration}"
1714             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1715             if (scenario == 'innerloop') {
1716                 contextString += " Innerloop"
1717                 triggerString += "\\W+Innerloop"
1718             }
1719             else {
1720                 contextString += " ${scenario}"
1721                 triggerString += "\\W+${scenario}"
1722             }
1723
1724             if (configuration == 'Debug') {
1725                 contextString += " Build"
1726                 triggerString += "\\W+Build"
1727             } else {
1728                 contextString += " Build and Test"
1729                 triggerString += "\\W+Build and Test"
1730             }
1731
1732             triggerString += ".*"
1733
1734             switch (os) {
1735                 case 'Ubuntu':
1736                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1737                         break
1738                     }
1739
1740                     if (scenario == 'innerloop') {
1741                         if (configuration == 'Checked') {
1742                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1743                         }
1744                     }
1745                     else {
1746                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1747                     }
1748                     break
1749
1750                 case 'Windows_NT':
1751                     if (architecture == "armlb") {
1752                         // Disable armlb windows jobs
1753                         break
1754                     }
1755                     switch (scenario) {
1756                         case 'innerloop':
1757                             // Only Checked is an innerloop trigger.
1758                             if (configuration == 'Checked')
1759                             {
1760                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1761                             }
1762                             break
1763                         case 'normal':
1764                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1765                             break
1766                         default:
1767                             // Stress jobs will use this code path.
1768                             if (isArmWindowsScenario(scenario)) {
1769                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1770                             }
1771                             break
1772                     }
1773                     break
1774                 default:
1775                     println("NYI os: ${os}");
1776                     assert false
1777                     break
1778             }
1779             break
1780         // editor brace matching: }
1781         case 'arm64': // editor brace matching: {
1782             // Set up a private trigger
1783             def contextString = "${os} ${architecture} Cross ${configuration}"
1784             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1785
1786             if (scenario == 'innerloop') {
1787                 contextString += " Innerloop"
1788                 triggerString += "\\W+Innerloop"
1789             }
1790             else {
1791                 contextString += " ${scenario}"
1792                 triggerString += "\\W+${scenario}"
1793             }
1794
1795             if (configuration == 'Debug') {
1796                 contextString += " Build"
1797                 triggerString += "\\W+Build"
1798             } else {
1799                 contextString += " Build and Test"
1800                 triggerString += "\\W+Build and Test"
1801             }
1802
1803             triggerString += ".*"
1804
1805             switch (os) {
1806                 case 'Ubuntu':
1807                 case 'Ubuntu16.04':
1808                     switch (scenario) {
1809                         case 'innerloop':
1810                             if (configuration == 'Debug' && !isFlowJob) {
1811                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1812                             }
1813
1814                             break
1815                         case 'normal':
1816                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1817                             break
1818                         default:
1819                             if (isR2RScenario(scenario)) {
1820                                 if (configuration == 'Checked' || configuration == 'Release') {
1821                                     def displayStr = getR2RDisplayName(scenario)
1822                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1823                                 }
1824                             }
1825                             break
1826                     }
1827                     break
1828
1829                 case 'Windows_NT':
1830                     // Triggers on the non-flow jobs aren't necessary here
1831                     if (!isFlowJob) {
1832                         break
1833                     }
1834
1835                     assert isArmWindowsScenario(scenario)
1836                     switch (scenario) {
1837                         case 'innerloop':
1838                             if (configuration == 'Checked') {
1839                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1840                             }
1841
1842                             break
1843                         case 'normal':
1844                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1845                             break
1846                         default:
1847                             // Stress jobs will use this code path.
1848                             if (isArmWindowsScenario(scenario)) {
1849                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1850                             }
1851                             break
1852                     }
1853                     break
1854                 default:
1855                     println("NYI os: ${os}");
1856                     assert false
1857                     break
1858             }
1859             break
1860
1861         // editor brace matching: }
1862         case 'x86': // editor brace matching: {
1863             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1864             if (os == 'Ubuntu') {
1865                 // Triggers on the non-flow jobs aren't necessary here
1866                 if (!isFlowJob) {
1867                     break
1868                 }
1869
1870                 // on-demand only for ubuntu x86
1871                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1872                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1873                 break
1874
1875             }
1876             switch (scenario) {
1877                 case 'innerloop':
1878                     if (configuration == 'Checked' || configuration == 'Release') {
1879                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1880                     }
1881                     break
1882
1883                 case 'normal':
1884                     if (configuration == 'Checked') {
1885                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1886                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1887                     }
1888                     break
1889
1890                 case 'ilrt':
1891                     if (configuration == 'Release') {
1892                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1893                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1894                     }
1895                     break
1896
1897                 case 'longgc':
1898                     if (configuration == 'Release') {
1899                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1900                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1901                     }
1902                     break
1903
1904                 case 'gcsimulator':
1905                     if (configuration == 'Release') {
1906                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1907                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1908                     }
1909                     break
1910
1911                 case 'standalone_gc':
1912                     if (configuration == 'Release' || configuration == 'Checked') {
1913                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1914                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1915                     }
1916                     break
1917
1918                 case 'illink':
1919                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1920                     break
1921
1922                 default:
1923                     if (isJitStressScenario(scenario)) {
1924                         def displayStr = getStressModeDisplayName(scenario)
1925                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1926                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1927                     }
1928                     else if (isR2RScenario(scenario)) {
1929                         if (configuration == 'Release' || configuration == 'Checked') {
1930                             def displayStr = getR2RDisplayName(scenario)
1931                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1932                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1933                         }
1934                     }
1935                     else {
1936                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1937                         assert false
1938                     }
1939                     break
1940
1941             }
1942             break
1943
1944          // editor brace matching: }
1945         case 'x64_arm64_altjit':
1946         case 'x86_arm_altjit': // editor brace matching: {
1947             assert (os == 'Windows_NT')
1948             switch (scenario) {
1949                 case 'normal':
1950                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1951                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1952                     break
1953                 default:
1954                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1955                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1956                     break
1957             }
1958             break
1959
1960         // editor brace matching: }
1961         default:
1962             println("Unknown architecture: ${architecture}");
1963             assert false
1964             break
1965     }
1966 }
1967
1968 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1969     def buildCommands = []
1970     def osGroup = getOSGroup(os)
1971     def lowerConfiguration = configuration.toLowerCase()
1972
1973     def priority = '1'
1974     if (scenario == 'innerloop') {
1975         priority = '0'
1976     }
1977
1978     def doCoreFxTesting = isCoreFxScenario(scenario)
1979
1980     // Calculate the build steps, archival, and xunit results
1981     switch (os) {
1982         case 'Windows_NT': // editor brace matching: {
1983             switch (architecture) {
1984                 case 'x64':
1985                 case 'x86':
1986                 case 'x86_arm_altjit':
1987                 case 'x64_arm64_altjit':
1988                     def arch = architecture
1989                     def buildOpts = ''
1990                     if (architecture == 'x86_arm_altjit') {
1991                         arch = 'x86'
1992                     }
1993                     else if (architecture == 'x64_arm64_altjit') {
1994                         arch = 'x64'
1995                     }
1996
1997                     if (scenario == 'formatting') {
1998                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1999                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2000                         break
2001                     }
2002
2003                     if (scenario == 'illink') {
2004                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2005                     }
2006
2007                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2008                     if ((lowerConfiguration == 'release') &&
2009                         (scenario in Constants.basicScenarios) &&
2010                         (architecture != 'x86_arm_altjit') &&
2011                         (architecture != 'x64_arm64_altjit')) {
2012
2013                         buildOpts += ' -enforcepgo'
2014                     }
2015
2016                     if (doCoreFxTesting) {
2017                         buildOpts += ' skiptests';
2018                     } else {
2019                         buildOpts += " -priority=${priority}"
2020                     }
2021
2022                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2023                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2024                     // 35 characters long.
2025
2026                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2027
2028                     if (!isBuildOnly) {
2029                         def runtestArguments = ''
2030                         def testOpts = 'collectdumps'
2031
2032                         if (isR2RScenario(scenario)) {
2033
2034                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2035                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2036                             // to cause the tests to be crossgen'ed.
2037
2038                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2039                                 testOpts += ' crossgenaltjit protononjit.dll'
2040                             } else {
2041                                 testOpts += ' crossgen'
2042                             }
2043
2044                             testOpts += ' runcrossgentests'
2045                         }
2046                         else if (scenario == 'jitdiff') {
2047                             testOpts += ' jitdisasm crossgen'
2048                         }
2049                         else if (scenario == 'ilrt') {
2050                             testOpts += ' ilasmroundtrip'
2051                         }
2052                         else if (isLongGc(scenario)) {
2053                             testOpts += " ${scenario} sequential"
2054                         }
2055                         else if (scenario == 'standalone_gc') {
2056                             testOpts += ' gcname clrgc.dll'
2057                         }
2058                         else if (scenario == 'illink') {
2059                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2060                         }
2061
2062                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2063                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2064                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2065                         // which is read by the xunit harness.)
2066                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2067                         {
2068                             def timeout = 1800000
2069                             testOpts += " timeout ${timeout}"
2070                         }
2071
2072                         // If we are running a stress mode, we should write out the set of key
2073                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2074
2075                         def envScriptPath = ''
2076                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2077                             def buildCommandsStr = ''
2078                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2079                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2080
2081                             if (isJitStressScenario(scenario)) {
2082                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2083                             }
2084                             else if (isR2RStressScenario(scenario)) {
2085                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2086                             }
2087
2088                             if (architecture == 'x86_arm_altjit') {
2089                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2090                             }
2091                             else if (architecture == 'x64_arm64_altjit') {
2092                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2093                             }
2094
2095                             envScriptFinalize(os, envScriptPath)
2096
2097                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2098                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2099                             // into a single script.
2100                             buildCommands += buildCommandsStr
2101                         }
2102                         else if (architecture == 'x86_arm_altjit') {
2103                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2104                         }
2105                         else if (architecture == 'x64_arm64_altjit') {
2106                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2107                         }
2108                         if (envScriptPath != '') {
2109                             testOpts += " TestEnv ${envScriptPath}"
2110                         }
2111
2112                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2113
2114                         if (doCoreFxTesting) {
2115                             def workspaceRelativeFxRoot = "_/fx"
2116                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2117
2118                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2119
2120                             // Archive and process (only) the test results
2121                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2122                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2123
2124                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2125                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2126                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2127                         }
2128                         else if (isGcReliabilityFramework(scenario)) {
2129                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2130                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2131                         }
2132                         else {
2133                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2134                         }
2135                     } // end if (!isBuildOnly)
2136
2137                     if (!doCoreFxTesting) {
2138                         // Run the rest of the build
2139                         // Build the mscorlib for the other OS's
2140                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2141                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2142
2143                         if (arch == 'x64') {
2144                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2145                         }
2146
2147                         // Zip up the tests directory so that we don't use so much space/time copying
2148                         // 10s of thousands of files around.
2149                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2150
2151                         if (!isJitStressScenario(scenario)) {
2152                             // For Windows, pull full test results and test drops for x86/x64.
2153                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2154                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2155                         }
2156
2157                         if (scenario == 'jitdiff') {
2158                             // retrieve jit-dasm output for base commit, and run jit-diff
2159                             if (!isBuildOnly) {
2160                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2161                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2162                             }
2163                         }
2164
2165                         if (!isBuildOnly) {
2166                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2167                         }
2168                     }
2169                     break
2170                 case 'armlb':
2171                 case 'arm':
2172                     assert isArmWindowsScenario(scenario)
2173
2174                     def buildArchitecture = 'arm'
2175
2176                     def buildOpts = ''
2177
2178                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2179                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2180                     if (architecture == 'armlb') {
2181                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2182                     }
2183
2184                     if (doCoreFxTesting) {
2185                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2186                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2187                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2188                         //
2189                         // buildOpts += ' skiptests';
2190                         buildOpts += " -priority=0"
2191                     } else {
2192                         buildOpts += " -priority=${priority}"
2193                     }
2194
2195                     // This is now a build only job. Do not run tests. Use the flow job.
2196                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2197
2198                     if (doCoreFxTesting) {
2199                         assert isBuildOnly
2200                         assert architecture == 'arm'
2201
2202                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2203                         // to determine the correct matching corefx version git commit hash.
2204                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2205
2206                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2207                         def envScriptPath = ''
2208                         def buildCommandsStr = ''
2209                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2210                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2211                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2212                         envScriptFinalize(os, envScriptPath)
2213                         buildCommands += buildCommandsStr
2214
2215                         def workspaceRelativeFxRootLinux = "_/fx"
2216                         def workspaceRelativeFxRootWin = "_\\fx"
2217                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2218
2219                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2220
2221                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2222                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2223                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2224
2225                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2226                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2227                     } else {
2228                         // Zip up the tests directory so that we don't use so much space/time copying
2229                         // 10s of thousands of files around.
2230                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2231
2232                         // Add archival.
2233                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2234                     }
2235                     break
2236                 case 'arm64':
2237                     assert isArmWindowsScenario(scenario)
2238
2239                     // This is now a build only job. Do not run tests. Use the flow job.
2240                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2241
2242                     // Zip up the tests directory so that we don't use so much space/time copying
2243                     // 10s of thousands of files around.
2244                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2245
2246                     // Add archival.
2247                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2248                     break
2249                 default:
2250                     println("Unknown architecture: ${architecture}");
2251                     assert false
2252                     break
2253             }
2254             break
2255         // end case 'Windows_NT'; editor brace matching: }
2256         case 'Ubuntu':
2257         case 'Ubuntu16.04':
2258         case 'Ubuntu16.10':
2259         case 'Debian8.4':
2260         case 'OSX10.12':
2261         case 'CentOS7.1':
2262         case 'RHEL7.2':
2263         case 'Tizen':
2264         case 'Fedora24': // editor brace matching: {
2265             switch (architecture) {
2266                 case 'x64':
2267                 case 'x86':
2268                     if (architecture == 'x86' && os == 'Ubuntu') {
2269                         // build and PAL test
2270                         def dockerImage = getDockerImageName(architecture, os, true)
2271                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2272                         dockerImage = getDockerImageName(architecture, os, false)
2273                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2274                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2275                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2276                         break
2277                     }
2278
2279                     if (scenario == 'formatting') {
2280                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2281                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2282                         break
2283                     }
2284
2285                     if (scenario == 'illink') {
2286                         assert(os == 'Ubuntu')
2287                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2288                     }
2289
2290                     if (!doCoreFxTesting) {
2291                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2292                         // only on supported OS platforms.
2293                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2294                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2295
2296                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2297                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2298
2299                         // Basic archiving of the build
2300                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2301                         // And pal tests
2302                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2303                     }
2304                     else {
2305                         // Corefx stress testing
2306                         assert os == 'Ubuntu'
2307                         assert architecture == 'x64'
2308                         assert lowerConfiguration == 'checked'
2309                         assert isJitStressScenario(scenario)
2310
2311                         // Build coreclr
2312                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2313
2314                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2315
2316                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2317                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2318                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2319                         buildCommands += envScriptCmds
2320
2321                         // Build and text corefx
2322                         def workspaceRelativeFxRoot = "_/fx"
2323                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2324
2325                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2326
2327                         // Archive and process (only) the test results
2328                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2329                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2330                     }
2331                     break
2332                 case 'arm64':
2333                     if (!doCoreFxTesting) {
2334                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2335
2336                         // HACK -- Arm64 does not have corefx jobs yet.
2337                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2338                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2339                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2340                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2341
2342                         // Basic archiving of the build
2343                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2344                     }
2345                     break
2346                 case 'armem':
2347                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2348                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2349
2350                     // default values for Ubuntu
2351                     def arm_abi = "arm"
2352                     def linuxCodeName = "trusty"
2353                     if (os == 'Ubuntu16.04') {
2354                         linuxCodeName = "xenial"
2355                     }
2356                     else if (os == 'Tizen') {
2357                         arm_abi = "armel"
2358                         linuxCodeName = "tizen"
2359                     }
2360
2361                     // Unzip the Windows test binaries first. Exit with 0
2362                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2363
2364                     // Unpack the corefx binaries
2365                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2366                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2367                     if (os != 'Tizen') {
2368                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2369                     }
2370                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2371                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2372                     if (os == 'Ubuntu') {
2373                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2374                     }
2375
2376                     // Call the ARM CI script to cross build and test using docker
2377                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2378                     --mode=docker \\
2379                     --${arm_abi} \\
2380                     --linuxCodeName=${linuxCodeName} \\
2381                     --buildConfig=${lowerConfiguration} \\
2382                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2383                     --coreFxBinDir=./bin/CoreFxBinDir \\
2384                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2385
2386                     // Basic archiving of the build, no pal tests
2387                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2388                     break
2389                 case 'arm':
2390                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2391                     assert (os == 'Ubuntu')
2392
2393                     // Add some useful information to the log file. Ignore return codes.
2394                     buildCommands += "uname -a || true"
2395
2396                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2397                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2398
2399                     def dockerImage = getDockerImageName(architecture, os, true)
2400                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm -e CAC_ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} "
2401
2402                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2403
2404                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2405                     // download the appropriate CoreFX packages.
2406                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2407                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2408
2409                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2410
2411                     // ZIP up for the test job (created in the flow job code):
2412                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2413                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2414                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2415                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2416
2417                     // These commands are assumed to be run from the root of the workspace.
2418                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2419                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2420
2421                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2422                     break
2423                 default:
2424                     println("Unknown architecture: ${architecture}");
2425                     assert false
2426                     break
2427             }
2428             break
2429         // editor brace matching: }
2430         default:
2431             println("Unknown os: ${os}");
2432             assert false
2433             break
2434     } // os
2435
2436     return buildCommands
2437 }
2438
2439 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2440 // Returns true if the job should be generated.
2441 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2442 {
2443     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2444     // All other scenarios do Pri-1 testing.
2445     if (scenario == 'innerloop' && !isPR) {
2446         return false
2447     }
2448
2449     // Tizen is only supported for armem architecture
2450     if (os == 'Tizen' && architecture != 'armem') {
2451         return false
2452     }
2453
2454     // Filter based on architecture.
2455
2456     switch (architecture) {
2457         case 'arm64':
2458         case 'arm':
2459             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2460                 return false
2461             }
2462             break
2463         case 'armem':
2464             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2465                 return false
2466             }
2467             break
2468         case 'armlb':
2469             // Do not create armlb jobs
2470             return false
2471         case 'x86_arm_altjit':
2472         case 'x64_arm64_altjit':
2473             if (os != 'Windows_NT') {
2474                 return false
2475             }
2476             break
2477         case 'x86':
2478             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2479                 return false
2480             }
2481             break
2482         case 'x64':
2483             // Everything implemented
2484             break
2485         default:
2486             println("Unknown architecture: ${architecture}")
2487             assert false
2488             break
2489     }
2490
2491     // Which (Windows) build only jobs are required?
2492
2493     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2494
2495     if (isBuildOnly) {
2496         switch (architecture) {
2497             case 'arm':
2498                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2499                 if (!isCoreFxScenario(scenario)) {
2500                     return false
2501                 }
2502                 break
2503             case 'x64':
2504             case 'x86':
2505                 if (!isNormalOrInnerloop) {
2506                     return false
2507                 }
2508                 break
2509             default:
2510                 return false
2511         }
2512     }
2513
2514     // Filter based on scenario.
2515
2516     if (isJitStressScenario(scenario)) {
2517         if (configuration != 'Checked') {
2518             return false
2519         }
2520
2521         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2522         if (!isEnabledOS) {
2523             return false
2524         }
2525
2526         switch (architecture) {
2527             case 'x64':
2528             case 'x86_arm_altjit':
2529             case 'x64_arm64_altjit':
2530                 break
2531
2532             case 'x86':
2533                 // x86 ubuntu: no stress modes
2534                 if (os == 'Ubuntu') {
2535                     return false
2536                 }
2537                 break
2538
2539             case 'arm':
2540                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2541                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2542                     return false
2543                 }
2544                 break
2545
2546             default:
2547                 // arm64, armlb: stress is handled through flow jobs.
2548                 // armem: no stress jobs for ARM emulator.
2549                 return false
2550         }
2551     }
2552     else if (isR2RScenario(scenario)) {
2553         if (os != 'Windows_NT') {
2554             return false
2555         }
2556         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2557         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2558             return false
2559         }
2560     }
2561     else {
2562         // Skip scenarios
2563         switch (scenario) {
2564             case 'ilrt':
2565                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2566                 // the default scenario build
2567                 if (os != 'Windows_NT') {
2568                     return false
2569                 }
2570                 // Only x64 for now
2571                 if (architecture != 'x64') {
2572                     return false
2573                 }
2574                 // Release only
2575                 if (configuration != 'Release') {
2576                     return false
2577                 }
2578                 break
2579             case 'jitdiff':
2580                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2581                     return false
2582                 }
2583                 if (architecture != 'x64') {
2584                     return false
2585                 }
2586                 if (configuration != 'Checked') {
2587                     return false
2588                 }
2589                 break
2590             case 'longgc':
2591             case 'gcsimulator':
2592                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2593                     return false
2594                 }
2595                 if (architecture != 'x64') {
2596                     return false
2597                 }
2598                 if (configuration != 'Release') {
2599                     return false
2600                 }
2601                 break
2602             case 'gc_reliability_framework':
2603             case 'standalone_gc':
2604                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2605                     return false
2606                 }
2607
2608                 if (architecture != 'x64') {
2609                     return false
2610                 }
2611
2612                 if (configuration != 'Release' && configuration != 'Checked') {
2613                     return false
2614                 }
2615                 break
2616             // We only run Windows and Ubuntu x64 Checked for formatting right now
2617             case 'formatting':
2618                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2619                     return false
2620                 }
2621                 if (architecture != 'x64') {
2622                     return false
2623                 }
2624                 if (configuration != 'Checked') {
2625                     return false
2626                 }
2627                 break
2628             case 'illink':
2629                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2630                     return false
2631                 }
2632                 if (architecture != 'x64' && architecture != 'x86') {
2633                     return false
2634                 }
2635                 break
2636             case 'normal':
2637                 // Nothing skipped
2638                 break
2639             case 'innerloop':
2640                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2641                     return false
2642                 }
2643                 break
2644             default:
2645                 println("Unknown scenario: ${scenario}")
2646                 assert false
2647                 break
2648         }
2649     }
2650
2651     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2652     // runtime behavior, not compile-time behavior, are not interesting.
2653     switch (architecture) {
2654         case 'x86_arm_altjit':
2655         case 'x64_arm64_altjit':
2656             if (isGCStressRelatedTesting(scenario)) {
2657                 return false
2658             }
2659             break
2660         default:
2661             break
2662     }
2663
2664     // The job was not filtered out, so we should generate it!
2665     return true
2666 }
2667
2668 Constants.allScenarios.each { scenario ->
2669     [true, false].each { isPR ->
2670         Constants.architectureList.each { architecture ->
2671             Constants.configurationList.each { configuration ->
2672                 Constants.osList.each { os ->
2673                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2674                     // and reset the os to Windows_NT
2675                     def isBuildOnly = false
2676                     if (os == 'Windows_NT_BuildOnly') {
2677                         isBuildOnly = true
2678                         os = 'Windows_NT'
2679                     }
2680
2681                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2682                         return
2683                     }
2684
2685                     // Calculate names
2686                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2687                     def folderName = getJobFolder(scenario)
2688
2689                     // Create the new job
2690                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2691                     addToViews(newJob, isPR, architecture, os)
2692
2693                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2694
2695                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2696                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2697                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2698
2699                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2700                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2701                     // artifacts produced.
2702                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2703                         // Define the Windows Tests and Corefx build job names
2704                         def lowerConfiguration = configuration.toLowerCase()
2705                         def WindowsTestsName = projectFolder + '/' +
2706                                                Utilities.getFullJobName(project,
2707                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2708                                                                         false)
2709                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2710                                            Utilities.getFolderName(branch)
2711
2712                         def arm_abi = 'arm'
2713                         def corefx_os = 'linux'
2714                         if (os == 'Tizen') {
2715                             arm_abi = 'armel'
2716                             corefx_os = 'tizen'
2717                         }
2718
2719                         // Let's use release CoreFX to test checked CoreCLR,
2720                         // because we do not generate checked CoreFX in CoreFX CI yet.
2721                         def corefx_lowerConfiguration = lowerConfiguration
2722                         if (lowerConfiguration == 'checked') {
2723                             corefx_lowerConfiguration = 'release'
2724                         }
2725
2726                         // Copy the Windows test binaries and the Corefx build binaries
2727                         newJob.with {
2728                             steps {
2729                                 copyArtifacts(WindowsTestsName) {
2730                                     includePatterns('bin/tests/tests.zip')
2731                                     buildSelector {
2732                                         latestSuccessful(true)
2733                                     }
2734                                 }
2735                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2736                                     includePatterns('bin/build.tar.gz')
2737                                     buildSelector {
2738                                         latestSuccessful(true)
2739                                     }
2740                                 }
2741                             } // steps
2742                         } // newJob.with
2743                     }
2744
2745                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2746
2747                     newJob.with {
2748                         steps {
2749                             if (os == 'Windows_NT') {
2750                                 buildCommands.each { buildCommand ->
2751                                     batchFile(buildCommand)
2752                                 }
2753                             }
2754                             else {
2755                                 buildCommands.each { buildCommand ->
2756                                     shell(buildCommand)
2757                                 }
2758                             }
2759                         } // steps
2760                     } // newJob.with
2761
2762                 } // os
2763             } // configuration
2764         } // architecture
2765     } // isPR
2766 } // scenario
2767
2768 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2769 // Returns the newly created job.
2770 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2771 {
2772     def osGroup = getOSGroup(os)
2773     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2774
2775     def jobFolder = getJobFolder(scenario)
2776     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2777         parameters {
2778             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2779         }
2780
2781         steps {
2782             // Set up the copies
2783
2784             // Coreclr build we are trying to test
2785             //
2786             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2787
2788             copyArtifacts(inputCoreCLRBuildName) {
2789                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2790                 buildSelector {
2791                     buildNumber('${CORECLR_BUILD}')
2792                 }
2793             }
2794
2795             if (isCoreFxScenario(scenario)) {
2796
2797                 // Only arm supported for corefx testing now.
2798                 assert architecture == 'arm'
2799
2800                 // Unzip CoreFx runtime
2801                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2802
2803                 // Unzip CoreFx tests.
2804                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2805
2806                 // Add the script to run the corefx tests
2807                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2808                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2809                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2810                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2811
2812             } else { // !isCoreFxScenario(scenario)
2813
2814                 // Unzip tests.
2815                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2816
2817                 def buildCommands = ""
2818
2819                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2820                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2821                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2822
2823                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2824                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2825
2826                 // For all jobs
2827                 addEnvVariable("CORE_ROOT", coreRootLocation)
2828                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2829                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2830
2831                 // ARM legacy backend; this is an altjit.
2832                 if (architecture == 'armlb') {
2833                     addEnvVariable("COMPlus_AltJit", "*")
2834                     addEnvVariable("COMPlus_AltJitNgen", "*")
2835                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2836                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2837                 }
2838
2839                 // If we are running a stress mode, we'll set those variables as well
2840                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2841                     def stressValues = null
2842                     if (isJitStressScenario(scenario)) {
2843                         stressValues = Constants.jitStressModeScenarios[scenario]
2844                     }
2845                     else {
2846                         stressValues = Constants.r2rStressScenarios[scenario]
2847                     }
2848
2849                     stressValues.each { key, value ->
2850                         addEnvVariable(key, value)
2851                     }
2852                 }
2853
2854                 if (isR2RScenario(scenario)) {
2855                     // Crossgen the framework assemblies.
2856                     buildCommands += """
2857 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2858 @goto skip_PrecompileAssembly
2859
2860 :PrecompileAssembly
2861 @REM Skip mscorlib since it is already precompiled.
2862 @if /I "%3" == "mscorlib.dll" exit /b 0
2863 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2864
2865 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2866 @if "%errorlevel%" == "-2146230517" (
2867     echo %2 is not a managed assembly.
2868 ) else if "%errorlevel%" == "-2146234344" (
2869     echo %2 is not a managed assembly.
2870 ) else if %errorlevel% neq 0 (
2871     echo Unable to precompile %2
2872 ) else (
2873     echo Precompiled %2
2874 )
2875 @exit /b 0
2876
2877 :skip_PrecompileAssembly
2878 """
2879
2880                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2881                     // crossgen on tests before running them.
2882                     addEnvVariable("RunCrossGen", "true")
2883                 } // isR2RScenario(scenario)
2884
2885                 // Create the smarty command
2886                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2887                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2888                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2889                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2890
2891                 if (architecture == 'armlb') {
2892                     addExclude("LEGACYJIT_FAIL")
2893                 }
2894
2895                 // Exclude tests based on scenario.
2896                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2897                     addArchSpecificExclude(architecture, excludeTag)
2898                 }
2899
2900                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2901                 if (scenario == 'innerloop') {
2902                     addExclude("pri1")
2903                 }
2904
2905                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2906                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2907                 addExclude("LONG_RUNNING")
2908
2909                 smartyCommand += "/lstFile Tests.lst"
2910
2911                 def testListArch = [
2912                     'arm64': 'arm64',
2913                     'arm': 'arm',
2914                     'armlb': 'arm'
2915                 ]
2916
2917                 def archLocation = testListArch[architecture]
2918
2919                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2920                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2921                 addCommand("${smartyCommand}")
2922
2923                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2924                 // However, we also need to remove all the variables that were set during this batch file, so we
2925                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2926                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2927                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2928                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2929                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2930                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2931
2932                 // ZIP up the smarty output, no matter what the smarty result.
2933                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2934
2935                 addCommand("echo %errorlevel%")
2936                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2937
2938                 // Use the smarty errorlevel as the script errorlevel.
2939                 addCommand("exit /b %__save_smarty_errorlevel%")
2940
2941                 batchFile(buildCommands)
2942             } // non-corefx testing
2943         } // steps
2944     } // job
2945
2946     if (!isCoreFxScenario(scenario)) {
2947         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2948
2949         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2950         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2951         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2952         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2953     }
2954
2955     return newJob
2956 }
2957
2958 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2959 // E.g., non-Windows tests.
2960 // Returns the newly created job.
2961 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2962 {
2963     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2964
2965     def osGroup = getOSGroup(os)
2966     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2967
2968     def testOpts = ''
2969     def useServerGC = false
2970
2971     // Enable Server GC for Ubuntu PR builds
2972     // REVIEW: why? Does this apply to all architectures? Why only PR?
2973     if (os == 'Ubuntu' && isPR) {
2974         testOpts += ' --useServerGC'
2975         useServerGC = true
2976     }
2977
2978     if (isR2RScenario(scenario)) {
2979
2980         testOpts += ' --crossgen --runcrossgentests'
2981
2982         if (scenario == 'r2r_jitstress1') {
2983             testOpts += ' --jitstress=1'
2984         }
2985         else if (scenario == 'r2r_jitstress2') {
2986             testOpts += ' --jitstress=2'
2987         }
2988         else if (scenario == 'r2r_jitstressregs1') {
2989             testOpts += ' --jitstressregs=1'
2990         }
2991         else if (scenario == 'r2r_jitstressregs2') {
2992             testOpts += ' --jitstressregs=2'
2993         }
2994         else if (scenario == 'r2r_jitstressregs3') {
2995             testOpts += ' --jitstressregs=3'
2996         }
2997         else if (scenario == 'r2r_jitstressregs4') {
2998             testOpts += ' --jitstressregs=4'
2999         }
3000         else if (scenario == 'r2r_jitstressregs8') {
3001             testOpts += ' --jitstressregs=8'
3002         }
3003         else if (scenario == 'r2r_jitstressregs0x10') {
3004             testOpts += ' --jitstressregs=0x10'
3005         }
3006         else if (scenario == 'r2r_jitstressregs0x80') {
3007             testOpts += ' --jitstressregs=0x80'
3008         }
3009         else if (scenario == 'r2r_jitstressregs0x1000') {
3010             testOpts += ' --jitstressregs=0x1000'
3011         }
3012         else if (scenario == 'r2r_jitminopts') {
3013             testOpts += ' --jitminopts'
3014         }
3015         else if (scenario == 'r2r_jitforcerelocs') {
3016             testOpts += ' --jitforcerelocs'
3017         }
3018         else if (scenario == 'r2r_gcstress15') {
3019             testOpts += ' --gcstresslevel=0xF'
3020         }
3021     }
3022     else if (scenario == 'jitdiff') {
3023         testOpts += ' --jitdisasm --crossgen'
3024     }
3025     else if (scenario == 'illink') {
3026         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3027     }
3028     else if (isLongGc(scenario)) {
3029         // Long GC tests behave very poorly when they are not
3030         // the only test running (many of them allocate until OOM).
3031         testOpts += ' --sequential'
3032
3033         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3034         // for running long GC and GCSimulator tests, respectively. We don't use them
3035         // here because using a playlist file produces much more readable output on the CI machines
3036         // and reduces running time.
3037         //
3038         // The Long GC playlist contains all of the tests that are
3039         // going to be run. The GCSimulator playlist contains all of
3040         // the GC simulator tests.
3041         if (scenario == 'longgc') {
3042             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3043         }
3044         else if (scenario == 'gcsimulator') {
3045             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3046         }
3047     }
3048     else if (isGcReliabilityFramework(scenario)) {
3049         testOpts += ' --build-overlay-only'
3050     }
3051     else if (scenario == 'standalone_gc') {
3052         if (osGroup == 'OSX') {
3053             testOpts += ' --gcname=libclrgc.dylib'
3054         }
3055         else if (osGroup == 'Linux') {
3056             testOpts += ' --gcname=libclrgc.so'
3057         }
3058         else {
3059             println("Unexpected OS group: ${osGroup} for os ${os}")
3060             assert false
3061         }
3062     }
3063
3064     def jobFolder = getJobFolder(scenario)
3065     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3066         parameters {
3067             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3068             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3069         }
3070
3071         steps {
3072             // Set up the copies
3073
3074             // Coreclr build containing the tests and mscorlib
3075             // pri1 jobs still need to copy windows_nt built tests
3076             assert inputTestsBuildName != null
3077             copyArtifacts(inputTestsBuildName) {
3078                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3079                 buildSelector {
3080                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3081                 }
3082             }
3083
3084             // Coreclr build we are trying to test
3085             //
3086             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3087
3088             copyArtifacts(inputCoreCLRBuildName) {
3089                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3090                 buildSelector {
3091                     buildNumber('${CORECLR_BUILD}')
3092                 }
3093             }
3094
3095             if (isUbuntuArmJob) {
3096                 // Add some useful information to the log file. Ignore return codes.
3097                 shell("uname -a || true")
3098             }
3099
3100             if (architecture == 'arm64') {
3101                 shell("mkdir -p ./bin/CoreFxBinDir")
3102                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3103                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3104             }
3105             else if (architecture == 'x86') {
3106                 shell("mkdir ./bin/CoreFxNative")
3107
3108                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3109
3110                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3111                     includePatterns('bin/build.tar.gz')
3112                     targetDirectory('bin/CoreFxNative')
3113                     buildSelector {
3114                         latestSuccessful(true)
3115                     }
3116                 }
3117
3118                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3119             }
3120
3121             // Unzip the tests first.  Exit with 0
3122             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3123             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3124
3125             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3126             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3127             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3128             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3129             // copied correctly.
3130             if (isUbuntuArmJob) {
3131                 def lowerConfiguration = configuration.toLowerCase()
3132                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3133                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3134             }
3135             else {
3136                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3137             }
3138
3139             // Execute the tests
3140             def runDocker = isNeedDocker(architecture, os, false)
3141             def dockerPrefix = ""
3142             def dockerCmd = ""
3143             if (runDocker) {
3144                 def dockerImage = getDockerImageName(architecture, os, false)
3145                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3146                 dockerCmd = dockerPrefix + "${dockerImage} "
3147             }
3148
3149             // If we are running a stress mode, we'll set those variables first
3150             if (isJitStressScenario(scenario)) {
3151                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3152                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3153                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3154                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3155                 shell("${envScriptCmds}")
3156                 testOpts += " --test-env=${scriptFileName}"
3157             }
3158
3159             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3160             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3161             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3162             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3163             if (isGCStressRelatedTesting(scenario)) {
3164                 if (architecture == 'x64') {
3165                     shell('./init-tools.sh')
3166                 }
3167             }
3168
3169             def runScript = "${dockerCmd}./tests/runtest.sh"
3170
3171             shell("""\
3172 ${runScript} \\
3173     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3174     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3175     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3176     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3177
3178             if (isGcReliabilityFramework(scenario)) {
3179                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3180                 if (useServerGC) {
3181                     if (runDocker) {
3182                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3183                     }
3184                     else {
3185                         shell("export COMPlus_gcServer=1")
3186                     }
3187                 }
3188
3189                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3190             }
3191         } // steps
3192     } // job
3193
3194     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3195     if (os in ['Ubuntu']) {
3196         SummaryBuilder summaries = new SummaryBuilder()
3197         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3198         summaries.emit(newJob)
3199     }
3200
3201     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3202     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3203
3204     return newJob
3205 }
3206
3207 // Create a test job that will be used by a flow job.
3208 // Returns the newly created job.
3209 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3210 {
3211     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3212
3213     def newJob = null
3214     if (windowsArmJob) {
3215         assert inputTestsBuildName == null
3216         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3217     } else {
3218         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3219     }
3220
3221     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3222
3223     addToViews(newJob, isPR, architecture, os)
3224
3225     if (scenario == 'jitdiff') {
3226         def osGroup = getOSGroup(os)
3227         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3228     }
3229
3230     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3231     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3232
3233     return newJob
3234 }
3235
3236 // Create a flow job to tie together a build job with the given test job.
3237 // Returns the new flow job.
3238 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3239 {
3240     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3241     // Linux CoreCLR test
3242     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3243     def jobFolder = getJobFolder(scenario)
3244
3245     def newFlowJob = null
3246
3247     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3248     if (windowsArmJob) {
3249
3250         assert inputTestsBuildName == null
3251
3252         // For Windows arm jobs there is no reason to build a parallel test job.
3253         // The product build supports building and archiving the tests.
3254
3255         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3256                         buildFlow("""\
3257 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3258
3259 // And then build the test build
3260 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3261 """)
3262         }
3263         JobReport.Report.addReference(inputCoreCLRBuildName)
3264         JobReport.Report.addReference(fullTestJobName)
3265     }
3266     else {
3267         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3268                         buildFlow("""\
3269 // Build the input jobs in parallel
3270 parallel (
3271 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3272 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3273 )
3274
3275 // And then build the test build
3276 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3277                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3278 """)
3279         }
3280         JobReport.Report.addReference(inputCoreCLRBuildName)
3281         JobReport.Report.addReference(inputTestsBuildName)
3282         JobReport.Report.addReference(fullTestJobName)
3283     }
3284
3285     addToViews(newFlowJob, isPR, architecture, os)
3286
3287     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3288
3289     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3290     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3291
3292     return newFlowJob
3293 }
3294
3295 // Determine if we should generate a flow job for the given parameters.
3296 // Returns true if the job should be generated.
3297 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3298 {
3299     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3300     // All other scenarios do Pri-1 testing.
3301     if (scenario == 'innerloop' && !isPR) {
3302         return false
3303     }
3304
3305     // Filter based on OS and architecture.
3306
3307     switch (architecture) {
3308         case 'arm64':
3309             if (os != "Ubuntu" && os != "Windows_NT") {
3310                 return false
3311             }
3312             break
3313         case 'armlb':
3314             if (os != 'Windows_NT') {
3315                 return false
3316             }
3317             // Do not create armlb windows jobs.
3318             return false
3319         case 'arm':
3320             if (os != "Ubuntu" && os != "Windows_NT") {
3321                 return false
3322             }
3323             break
3324         case 'x86':
3325             if (os != "Ubuntu") {
3326                 return false
3327             }
3328             break
3329         case 'x64':
3330             if (!(os in Constants.crossList)) {
3331                 return false
3332             }
3333             if (os == "Windows_NT") {
3334                 return false
3335             }
3336             break
3337         case 'armem':
3338         case 'x86_arm_altjit':
3339         case 'x64_arm64_altjit':
3340             // No flow jobs
3341             return false
3342         default:
3343             println("Unknown architecture: ${architecture}")
3344             assert false
3345             break
3346     }
3347
3348     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3349
3350     // Filter based on scenario in OS.
3351
3352     if (os == 'Windows_NT') {
3353         if (!isArmWindowsScenario(scenario)) {
3354             return false
3355         }
3356     }
3357     else {
3358         // Non-Windows
3359         if (architecture == 'arm64') {
3360             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3361                 return false
3362             }
3363         }
3364         else if (architecture == 'arm') {
3365             if (!(scenario in Constants.validLinuxArmScenarios)) {
3366                 return false
3367             }
3368         }
3369         else if (architecture == 'x86') {
3370             // Linux/x86 only want innerloop and default test
3371             if (!isNormalOrInnerloop) {
3372                 return false
3373             }
3374         }
3375     }
3376
3377     // For CentOS, we only want Checked/Release builds.
3378     if (os == 'CentOS7.1') {
3379         if (configuration != 'Checked' && configuration != 'Release') {
3380             return false
3381         }
3382         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3383             return false
3384         }
3385     }
3386
3387     // For RedHat and Debian, we only do Release builds.
3388     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3389         if (configuration != 'Release') {
3390             return false
3391         }
3392         if (!isNormalOrInnerloop) {
3393             return false
3394         }
3395     }
3396
3397     // Next, filter based on scenario.
3398
3399     if (isJitStressScenario(scenario)) {
3400         if (configuration != 'Checked') {
3401             return false
3402         }
3403
3404         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3405         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3406             return false
3407         }
3408     }
3409     else if (isR2RBaselineScenario(scenario)) {
3410         if (configuration != 'Checked' && configuration != 'Release') {
3411             return false
3412         }
3413     }
3414     else if (isR2RStressScenario(scenario)) {
3415         if (configuration != 'Checked') {
3416             return false
3417         }
3418     }
3419     else {
3420         // Skip scenarios
3421         switch (scenario) {
3422             case 'ilrt':
3423             case 'longgc':
3424             case 'gcsimulator':
3425                 // Long GC tests take a long time on non-Release builds
3426                 // ilrt is also Release only
3427                 if (configuration != 'Release') {
3428                     return false
3429                 }
3430                 break
3431
3432             case 'jitdiff':
3433                 if (configuration != 'Checked') {
3434                     return false
3435                 }
3436                 break
3437
3438             case 'gc_reliability_framework':
3439             case 'standalone_gc':
3440                 if (configuration != 'Release' && configuration != 'Checked') {
3441                     return false
3442                 }
3443                 break
3444
3445             case 'formatting':
3446                 return false
3447             case 'illink':
3448                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3449                     return false
3450                 }
3451                 break
3452
3453             case 'normal':
3454                 // Nothing skipped
3455                 break
3456
3457             case 'innerloop':
3458                 // Nothing skipped
3459                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3460                     return false
3461                 }
3462                 break
3463
3464             default:
3465                 println("Unknown scenario: ${scenario}")
3466                 assert false
3467                 break
3468         }
3469     }
3470
3471     // The job was not filtered out, so we should generate it!
3472     return true
3473 }
3474
3475 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3476 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3477 Constants.allScenarios.each { scenario ->
3478     [true, false].each { isPR ->
3479         Constants.architectureList.each { architecture ->
3480             Constants.configurationList.each { configuration ->
3481                 Constants.osList.each { os ->
3482
3483                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3484                         return
3485                     }
3486
3487                     // Figure out the job name of the CoreCLR build the test will depend on.
3488
3489                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3490                     def inputCoreCLRBuildIsBuildOnly = false
3491                     if (isCoreFxScenario(scenario)) {
3492                         // Every CoreFx test depends on its own unique build.
3493                         inputCoreCLRBuildScenario = scenario
3494                         inputCoreCLRBuildIsBuildOnly = true
3495                     }
3496                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3497                     def inputCoreCLRBuildName = projectFolder + '/' +
3498                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3499
3500                     // Figure out the name of the build job that the test job will depend on.
3501                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3502                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3503
3504                     def inputTestsBuildName = null
3505
3506                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3507                     if (!windowsArmJob) {
3508                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3509
3510                         def inputTestsBuildArch = architecture
3511                         if (architecture == "arm64") {
3512                             // Use the x64 test build for arm64 unix
3513                             inputTestsBuildArch = "x64"
3514                         }
3515                         else if (architecture == "arm") {
3516                             // Use the x86 test build for arm unix
3517                             inputTestsBuildArch = "x86"
3518                         }
3519
3520                         def inputTestsBuildIsBuildOnly = true
3521
3522                         inputTestsBuildName = projectFolder + '/' +
3523                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3524                     }
3525
3526                     // =============================================================================================
3527                     // Create the test job
3528                     // =============================================================================================
3529
3530                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3531
3532                     // =============================================================================================
3533                     // Create a build flow to join together the build and tests required to run this test.
3534                     // =============================================================================================
3535
3536                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3537                         // Do not create the flow job for RHEL jobs.
3538                         return
3539                     }
3540
3541                     def fullTestJobName = projectFolder + '/' + testJob.name
3542                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3543
3544                 } // os
3545             } // configuration
3546         } // architecture
3547     } // isPR
3548 } // scenario
3549
3550 JobReport.Report.generateJobReport(out)
3551
3552 // Make the call to generate the help job
3553 Utilities.createHelperJob(this, project, branch,
3554     "Welcome to the ${project} Repository",  // This is prepended to the help message
3555     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3556
3557 Utilities.addCROSSCheck(this, project, branch)