netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213     ]
 214
 215     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 216     // to Tests.lst file categories to exclude.
 217     //
 218     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 219     // and with the same values, with some commented out, for easier maintenance.
 220     //
 221     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 222     //
 223     def static validArmWindowsScenarios = [
 224                'innerloop':                              [],
 225                'normal':                                 [],
 226                // 'ilrt'
 227                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 228                // 'longgc'
 229                // 'formatting'
 230                // 'gcsimulator'
 231                // 'jitdiff'
 232                // 'standalone_gc'
 233                // 'gc_reliability_framework'
 234                // 'illink'
 235                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 246                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 248                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 249                'tieredcompilation':                      [],
 250                'forcerelocs':                            [],
 251                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 270                // 'jitsse2only'                          // Only relevant to xarch
 271                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 272                // 'jitincompletehwintrinsic'
 273                // 'jitx86hwintrinsicnoavx'
 274                // 'jitx86hwintrinsicnoavx2'
 275                // 'jitx86hwintrinsicnosimd'
 276                // 'jitnox86hwintrinsic'
 277                'corefx_baseline':                        [],    // corefx tests don't use smarty
 278                'corefx_minopts':                         [],    // corefx tests don't use smarty
 279                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 280                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 281                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 283                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 284                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 285                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 286                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 290                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 291                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 292                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 293                'heapverify1':                            [],
 294                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 295                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 297                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 300
 301                //
 302                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 303                //
 304
 305                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 306                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 324     ]
 325
 326     def static validLinuxArm64Scenarios = [
 327                'innerloop',
 328                'normal',
 329                'r2r',
 330                'gcstress0x3',
 331                'gcstress0xc'
 332     ]
 333
 334     def static validLinuxArmScenarios = [
 335                'innerloop',
 336                'normal',
 337                'r2r',
 338                'r2r_jitstress1',
 339                'r2r_jitstress2',
 340                'r2r_jitstressregs1',
 341                'r2r_jitstressregs2',
 342                'r2r_jitstressregs3',
 343                'r2r_jitstressregs4',
 344                'r2r_jitstressregs8',
 345                'r2r_jitstressregs0x10',
 346                'r2r_jitstressregs0x80',
 347                'r2r_jitstressregs0x1000',
 348                'r2r_jitminopts',
 349                'r2r_jitforcerelocs',
 350                'r2r_gcstress15',
 351                'minopts',
 352                'forcerelocs',
 353                'jitstress1',
 354                'jitstress2',
 355                'jitstressregs1',
 356                'jitstressregs2',
 357                'jitstressregs3',
 358                'jitstressregs4',
 359                'jitstressregs8',
 360                'jitstressregs0x10',
 361                'jitstressregs0x80',
 362                'jitstressregs0x1000',
 363                'jitstress2_jitstressregs1',
 364                'jitstress2_jitstressregs2',
 365                'jitstress2_jitstressregs3',
 366                'jitstress2_jitstressregs4',
 367                'jitstress2_jitstressregs8',
 368                'jitstress2_jitstressregs0x10',
 369                'jitstress2_jitstressregs0x80',
 370                'jitstress2_jitstressregs0x1000',
 371                'tailcallstress',
 372                'gcstress0x3',
 373                'gcstress0xc',
 374                'zapdisable',
 375                'heapverify1',
 376                'gcstress0xc_zapdisable',
 377                'gcstress0xc_zapdisable_jitstress2',
 378                'gcstress0xc_zapdisable_heapverify1',
 379                'gcstress0xc_jitstress1',
 380                'gcstress0xc_jitstress2',
 381                'gcstress0xc_minopts_heapverify1'
 382     ]
 383
 384     def static configurationList = ['Debug', 'Checked', 'Release']
 385
 386     // This is the set of architectures
 387     // Some of these are pseudo-architectures:
 388     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 389     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 390     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 391     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 392     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 393
 394     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 395     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 396 }
 397
 398 // **************************************************************
 399 // Create some specific views
 400 //
 401 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 402 // views based on a single regular expression. These views will be generated by adding a
 403 // specific set of jobs to them.
 404 //
 405 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 406 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 407 // those are required, don't add them (which simplifies the view pages, as well).
 408 // **************************************************************
 409
 410 class Views {
 411     def static MergeJobView = null
 412     def static PeriodicJobView = null
 413     def static ArchitectureViews = [:]
 414     def static OSViews = [:]
 415 }
 416
 417 // MergeJobView: include all jobs that execute when a PR change is merged.
 418 Views.MergeJobView = listView('Merge') {
 419     recurse()
 420     columns {
 421         status()
 422         weather()
 423         name()
 424         lastSuccess()
 425         lastFailure()
 426         lastDuration()
 427         buildButton()
 428     }
 429 }
 430
 431 // PeriodicJobView: include all jobs that execute on a schedule
 432 Views.PeriodicJobView = listView('Periodic') {
 433     recurse()
 434     columns {
 435         status()
 436         weather()
 437         name()
 438         lastSuccess()
 439         lastFailure()
 440         lastDuration()
 441         buildButton()
 442     }
 443 }
 444
 445 // Create a view for non-PR jobs for each architecture.
 446 Constants.architectureList.each { architecture ->
 447     Views.ArchitectureViews[architecture] = listView(architecture) {
 448         recurse()
 449         columns {
 450             status()
 451             weather()
 452             name()
 453             lastSuccess()
 454             lastFailure()
 455             lastDuration()
 456             buildButton()
 457         }
 458     }
 459 }
 460
 461 // Create a view for non-PR jobs for each OS.
 462 Constants.osList.each { os ->
 463     // Don't create one for the special 'Windows_NT_BuildOnly'
 464     if (os == 'Windows_NT_BuildOnly') {
 465         return
 466     }
 467     Views.OSViews[os] = listView(os) {
 468         recurse()
 469         columns {
 470             status()
 471             weather()
 472             name()
 473             lastSuccess()
 474             lastFailure()
 475             lastDuration()
 476             buildButton()
 477         }
 478     }
 479 }
 480
 481 def static addToMergeView(def job) {
 482     Views.MergeJobView.with {
 483         jobs {
 484             name(job.name)
 485         }
 486     }
 487 }
 488
 489 def static addToPeriodicView(def job) {
 490     Views.PeriodicJobView.with {
 491         jobs {
 492             name(job.name)
 493         }
 494     }
 495 }
 496
 497 def static addToViews(def job, def isPR, def architecture, def os) {
 498     if (isPR) {
 499         // No views want PR jobs currently.
 500         return
 501     }
 502
 503     // Add to architecture view.
 504     Views.ArchitectureViews[architecture].with {
 505         jobs {
 506             name(job.name)
 507         }
 508     }
 509
 510     // Add to OS view.
 511     Views.OSViews[os].with {
 512         jobs {
 513             name(job.name)
 514         }
 515     }
 516 }
 517
 518 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 519     addToPeriodicView(job)
 520     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 521 }
 522
 523 def static addGithubPushTriggerHelper(def job) {
 524     addToMergeView(job)
 525     Utilities.addGithubPushTrigger(job)
 526 }
 527
 528
 529 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 530     assert os instanceof String
 531     assert architecture instanceof String
 532
 533     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 534     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 535
 536     if (!(architecture in armArches)) {
 537         assert options == null
 538         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 539
 540         return
 541     }
 542
 543     // This is an arm(64) job.
 544     //
 545     // There are several options.
 546     //
 547     // Windows_NT
 548     //
 549     // Arm32 (Build) -> latest-arm64
 550     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 551     // Arm32 (Test)  -> arm64-windows_nt
 552     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 553     //
 554     // Arm64 (Build) -> latest-arm64
 555     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 556     // Arm64 (Test)  -> arm64-windows_nt
 557     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 558     //
 559     // Ubuntu
 560     //
 561     // Arm32 emulator (Build, Test) -> arm-cross-latest
 562     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 563     //
 564     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 565     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 566     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 567     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 568     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 569     //       |-> os == "Ubuntu" && (architecture == "arm")
 570     //
 571     // Arm64 (Build) -> arm64-cross-latest
 572     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 573     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 574     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 575     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 576     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 577
 578     // This has to be a arm arch
 579     assert architecture in armArches
 580     if (os == "Windows_NT") {
 581         // arm32/arm64 Windows jobs share the same machines for now
 582         def isBuild = options['use_arm64_build_machine'] == true
 583
 584         if (isBuild == true) {
 585             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 586         } else {
 587             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 588         }
 589     } else {
 590         assert os != 'Windows_NT'
 591         assert os in supportedArmLinuxOs
 592
 593         if (architecture == 'arm64') {
 594             if ((options != null) && (options['is_build_only'] == true)) {
 595                 // Arm64 Linux build machine
 596                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 597             } else {
 598                 // Arm64 Linux test machines
 599                 if ((options != null) && (options['large_pages'] == true)) {
 600                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 601                 } else {
 602                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 603                 }
 604             }
 605         }
 606         else if (architecture == 'armem') {
 607             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 608             // using Docker.
 609             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 610         }
 611         else {
 612             // arm Ubuntu on hardware.
 613             assert (architecture == 'arm') && (os == 'Ubuntu')
 614             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 615             def isBuild = (options != null) && (options['is_build_job'] == true)
 616             if (isFlow || isBuild) {
 617                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 618                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 619                 // arm hardware.
 620                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 621             } else {
 622                 // arm Ubuntu test machine
 623                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 624                 // Utilities.setMachineAffinity. Just add the machine affinity
 625                 // manually. We specify the Helix queue name here.
 626                 job.with {
 627                     label('ubuntu.1404.arm32.open')
 628                 }
 629             }
 630         }
 631     }
 632 }
 633
 634 // setJobMachineAffinity: compute the machine affinity options for a job,
 635 // then set the job with those affinity options.
 636 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 637 {
 638     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 639            (!isBuildJob && isTestJob  && !isFlowJob) ||
 640            (!isBuildJob && !isTestJob && isFlowJob)
 641
 642     def affinityOptions = null
 643     def affinityArchitecture = architecture
 644
 645     if (os == "Windows_NT") {
 646         if (architecture in Constants.armWindowsCrossArchitectureList) {
 647             if (isBuildJob) {
 648                 affinityOptions = [ "use_arm64_build_machine" : true ]
 649             } else if (isTestJob) {
 650                 affinityOptions = [ "use_arm64_build_machine" : false ]
 651             } else if (isFlowJob) {
 652                 // For the flow jobs set the machine affinity as x64
 653                 affinityArchitecture = 'x64'
 654             }
 655         }
 656     }
 657     else {
 658         if (architecture == 'arm64') {
 659             if (isBuildJob) {
 660                 affinityOptions = ['is_build_only': true]
 661             } else if (isTestJob) {
 662                 affinityOptions = [ "large_pages" : false ]
 663             }
 664         }
 665         else if (architecture == 'arm') {
 666             if (isBuildJob) {
 667                 affinityOptions = ['is_build_job': true]
 668             } else if (isFlowJob) {
 669                 affinityOptions = ['is_flow_job': true]
 670             }
 671         }
 672     }
 673
 674     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 675 }
 676
 677 def static isGCStressRelatedTesting(def scenario) {
 678     // The 'r2r_gcstress15' scenario is a basic scenario.
 679     // Detect it and make it a GCStress related.
 680     if (scenario == 'r2r_gcstress15')
 681     {
 682         return true;
 683     }
 684
 685     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 686     def scenarioName = scenario.toLowerCase()
 687     def isGCStressTesting = false
 688     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 689         if (k in gcStressTestEnvVars) {
 690             isGCStressTesting = true;
 691         }
 692     }
 693     return isGCStressTesting
 694 }
 695
 696 def static isCoreFxScenario(def scenario) {
 697     def corefx_prefix = 'corefx_'
 698     if (scenario.length() < corefx_prefix.length()) {
 699         return false
 700     }
 701     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 702 }
 703
 704 def static isR2RBaselineScenario(def scenario) {
 705     return (scenario == 'r2r')
 706 }
 707
 708 def static isR2RStressScenario(def scenario) {
 709     return Constants.r2rStressScenarios.containsKey(scenario)
 710 }
 711
 712 def static isR2RScenario(def scenario) {
 713     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 714 }
 715
 716 def static isJitStressScenario(def scenario) {
 717     return Constants.jitStressModeScenarios.containsKey(scenario)
 718 }
 719
 720 def static isLongGc(def scenario) {
 721     return (scenario == 'longgc' || scenario == 'gcsimulator')
 722 }
 723
 724 def static isJitDiff(def scenario) {
 725     return (scenario == 'jitdiff')
 726 }
 727
 728 def static isGcReliabilityFramework(def scenario) {
 729     return (scenario == 'gc_reliability_framework')
 730 }
 731
 732 def static isArmWindowsScenario(def scenario) {
 733     return Constants.validArmWindowsScenarios.containsKey(scenario)
 734 }
 735
 736 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 737     if (isBuildOnly == true) {
 738         os = 'Windows_NT_BuildOnly'
 739     }
 740
 741     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 742
 743     if (validOsPrTriggerArchConfigs == null) {
 744         return false
 745     }
 746
 747     if (validOsPrTriggerArchConfigs[architecture] != null) {
 748         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 749
 750         if (!(configuration in validOsPrTriggerConfigs)) {
 751             return false
 752         }
 753     } else {
 754         return false
 755     }
 756
 757     return true
 758 }
 759
 760 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 761     // 2 hours (120 minutes) is the default timeout
 762     def timeout = 120
 763     def innerLoop = (scenario == "innerloop")
 764
 765     if (!innerLoop) {
 766         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 767         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 768         timeout = 240
 769     }
 770
 771     if (!isBuildOnly) {
 772         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 773         if (isGCStressRelatedTesting(scenario)) {
 774             timeout = 4320
 775         }
 776         else if (isCoreFxScenario(scenario)) {
 777             timeout = 360
 778         }
 779         else if (isJitStressScenario(scenario)) {
 780             timeout = 300
 781         }
 782         else if (isR2RBaselineScenario(scenario)) {
 783             timeout = 240
 784         }
 785         else if (isLongGc(scenario)) {
 786             timeout = 1440
 787         }
 788         else if (isJitDiff(scenario)) {
 789             timeout = 240
 790         }
 791         else if (isGcReliabilityFramework(scenario)) {
 792             timeout = 1440
 793         }
 794         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 795             timeout = 240
 796         }
 797
 798         if (architecture == 'arm') {
 799             // ARM32 machines are particularly slow.
 800             timeout += 120
 801         }
 802     }
 803
 804     if (configuration == 'Debug') {
 805         // Debug runs can be very slow. Add an hour.
 806         timeout += 60
 807     }
 808
 809     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 810         // AltJit runs compile all methods twice.
 811         timeout *= 2
 812     }
 813
 814     // If we've changed the timeout from the default, set it in the job.
 815
 816     if (timeout != 120) {
 817         Utilities.setJobTimeout(newJob, timeout)
 818     }
 819 }
 820
 821 def static getJobFolder(def scenario) {
 822     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 823         return 'jitstress'
 824     }
 825     if (scenario == 'illink') {
 826         return 'illink'
 827     }
 828     return ''
 829 }
 830
 831 def static getStressModeDisplayName(def scenario) {
 832     def displayStr = ''
 833     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 834         def prefixLength = 'COMPlus_'.length()
 835         if (k.length() >= prefixLength) {
 836             def modeName = k.substring(prefixLength, k.length())
 837             displayStr += ' ' + modeName + '=' + v
 838         }
 839     }
 840
 841     if (isCoreFxScenario(scenario)) {
 842         displayStr = ('CoreFx ' + displayStr).trim()
 843     }
 844
 845     return displayStr
 846 }
 847
 848 def static getR2RDisplayName(def scenario) {
 849     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 850     def displayStr = scenario
 851     def prefixLength = 'r2r_'.length()
 852     if (displayStr.length() >= prefixLength) {
 853         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 854     } else if (scenario == 'r2r') {
 855         displayStr = "R2R"
 856     }
 857     return displayStr
 858 }
 859
 860 //
 861 // Functions to create an environment script.
 862 //      envScriptCreate -- initialize the script (call first)
 863 //      envScriptFinalize -- finalize the script (call last)
 864 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 865 //      envScriptAppendExistingScript -- append an existing script to the generated script
 866 //
 867 // Each script returns a string of commands. Concatenate all the strings together before
 868 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 869 //
 870
 871 // Initialize the environment setting script.
 872 def static envScriptCreate(def os, def stepScriptLocation) {
 873     def stepScript = ''
 874     if (os == 'Windows_NT') {
 875         stepScript += "echo Creating TestEnv script\r\n"
 876         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 877
 878         // Create at least an empty script.
 879         stepScript += "echo. > ${stepScriptLocation}\r\n"
 880     }
 881     else {
 882         stepScript += "echo Creating environment setting script\n"
 883         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 884     }
 885
 886     return stepScript
 887 }
 888
 889 // Generates the string for setting stress mode variables.
 890 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 891     def stepScript = ''
 892     if (os == 'Windows_NT') {
 893         stressModeVars.each{ k, v ->
 894             // Write out what we are writing to the script file
 895             stepScript += "echo Setting ${k}=${v}\r\n"
 896             // Write out the set itself to the script file`
 897             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 898         }
 899     }
 900     else {
 901         stressModeVars.each{ k, v ->
 902             // Write out what we are writing to the script file
 903             stepScript += "echo Setting ${k}=${v}\n"
 904             // Write out the set itself to the script file`
 905             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 906         }
 907     }
 908
 909     return stepScript
 910 }
 911
 912 // Append an existing script to an environment script.
 913 // Returns string of commands to do this.
 914 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 915     assert (os == 'Windows_NT')
 916     def stepScript = ''
 917
 918     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 919     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 920
 921     return stepScript
 922 }
 923
 924 // Finalize an environment setting script.
 925 // Returns string of commands to do this.
 926 def static envScriptFinalize(def os, def stepScriptLocation) {
 927     def stepScript = ''
 928
 929     if (os == 'Windows_NT') {
 930         // Display the resulting script. This is useful when looking at the output log file.
 931         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 932         stepScript += "type ${stepScriptLocation}\r\n"
 933     }
 934     else {
 935         stepScript += "chmod +x ${stepScriptLocation}\n"
 936     }
 937
 938     return stepScript
 939 }
 940
 941 def static isNeedDocker(def architecture, def os, def isBuild) {
 942     if (isBuild) {
 943         if (architecture == 'x86' && os == 'Ubuntu') {
 944             return true
 945         }
 946         else if (architecture == 'armem') {
 947             return true
 948         }
 949         else if (architecture == 'arm') {
 950             if (os == 'Ubuntu') {
 951                 return true
 952             }
 953         }
 954     }
 955     else {
 956         if (architecture == 'x86' && os == 'Ubuntu') {
 957             return true
 958         }
 959     }
 960     return false
 961 }
 962
 963 def static getDockerImageName(def architecture, def os, def isBuild) {
 964     // We must change some docker private images to official later
 965     if (isBuild) {
 966         if (architecture == 'x86' && os == 'Ubuntu') {
 967             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 968         }
 969         else if (architecture == 'armem') {
 970             if (os == 'Ubuntu') {
 971                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180405193556"
 972             }
 973             else if (os == 'Ubuntu16.04') {
 974                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180404203310"
 975             }
 976             else if (os == 'Tizen') {
 977                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 978             }
 979         }
 980         else if (architecture == 'arm') {
 981             if (os == 'Ubuntu') {
 982                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180405193556"
 983             }
 984         }
 985     }
 986     else {
 987         if (architecture == 'x86' && os == 'Ubuntu') {
 988             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 989         }
 990     }
 991     println("Unknown architecture to use docker: ${architecture} ${os}");
 992     assert false
 993 }
 994
 995
 996 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
 997 def static jobRequiresLimitedHardware(def architecture, def os) {
 998     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
 999         // These test jobs require ARM64 hardware
1000         return true
1001     }
1002     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1003         // These test jobs require Linux/arm32 hardware
1004         return true
1005     }
1006     else {
1007         return false
1008     }
1009 }
1010
1011 // Calculates the name of the build job based on some typical parameters.
1012 //
1013 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1014     // If the architecture is x64, do not add that info into the build name.
1015     // Need to change around some systems and other builds to pick up the right builds
1016     // to do that.
1017
1018     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1019     if (isBuildOnly) {
1020         suffix += '_bld'
1021     }
1022     def baseName = ''
1023     switch (architecture) {
1024         case 'x64':
1025             if (scenario == 'normal') {
1026                 // For now we leave x64 off of the name for compatibility with other jobs
1027                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1028             }
1029             else if (scenario == 'formatting') {
1030                 // we don't care about the configuration for the formatting job. It runs all configs
1031                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1032             }
1033             else {
1034                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1035             }
1036             break
1037         case 'arm64':
1038             if (os.toLowerCase() == "windows_nt") {
1039                 // These are cross builds
1040                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1041             }
1042             else {
1043                 // Defaults to a small page size set of machines.
1044                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1045             }
1046             break
1047         case 'armem':
1048             // These are cross builds
1049             if (os == 'Tizen') {
1050                 // ABI: softfp
1051                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1052             }
1053             else {
1054                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1055             }
1056             break
1057         case 'armlb':
1058         case 'arm':
1059             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1060             break
1061         case 'x86':
1062         case 'x86_arm_altjit':
1063         case 'x64_arm64_altjit':
1064             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1065             break
1066         default:
1067             println("Unknown architecture: ${architecture}");
1068             assert false
1069             break
1070     }
1071
1072     return baseName + suffix
1073 }
1074
1075 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1076
1077     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1078     if (os == 'Windows_NT') {
1079         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1080             if (!(branch in Constants.WindowsArm64Branches)) {
1081                 return
1082             }
1083         }
1084     }
1085
1086     if ((architecture == 'arm') && (os != 'Windows_NT') && isGCStressRelatedTesting(scenario)) {
1087         // Non-Windows Arm GCStress jobs currently don't get cron or push triggers (until they are functional).
1088         // See https://github.com/dotnet/coreclr/issues/17241.
1089         return
1090     }
1091
1092     // Check scenario.
1093     switch (scenario) {
1094         case 'innerloop':
1095             break
1096         case 'normal':
1097             switch (architecture) {
1098                 case 'x64':
1099                 case 'x86':
1100                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1101                         addPeriodicTriggerHelper(job, '@daily')
1102                     }
1103                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     break
1107                 case 'arm':
1108                     if (os == 'Windows_NT') {
1109                         addGithubPushTriggerHelper(job)
1110                     }
1111                     else {
1112                         // Currently no push triggers, with limited arm Linux hardware.
1113                         // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1114                         assert os == 'Ubuntu'
1115                         if (isFlowJob) {
1116                             addPeriodicTriggerHelper(job, '@daily')
1117                         }
1118                     }
1119                     break
1120                 case 'armem':
1121                 case 'armlb':
1122                 case 'x86_arm_altjit':
1123                 case 'x64_arm64_altjit':
1124                     addGithubPushTriggerHelper(job)
1125                     break
1126                 case 'arm64':
1127                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1128                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1129                     break
1130                 default:
1131                     println("Unknown architecture: ${architecture}");
1132                     assert false
1133                     break
1134             }
1135             break
1136         case 'r2r':
1137             assert !(os in bidailyCrossList)
1138             // r2r gets a push trigger for checked/release
1139             if (configuration == 'Checked' || configuration == 'Release') {
1140                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1141                 if (architecture == 'x64' && os != 'OSX10.12') {
1142                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1143                     if (isFlowJob || os == 'Windows_NT') {
1144                         addGithubPushTriggerHelper(job)
1145                     }
1146                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1147                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1148                     if (isFlowJob) {
1149                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1150                     }
1151                 }
1152                 // For x86, only add per-commit jobs for Windows
1153                 else if (architecture == 'x86') {
1154                     if (os == 'Windows_NT') {
1155                         addGithubPushTriggerHelper(job)
1156                     }
1157                 }
1158                 // arm64 r2r jobs should only run daily.
1159                 else if (architecture == 'arm64') {
1160                     if (os == 'Windows_NT') {
1161                         addPeriodicTriggerHelper(job, '@daily')
1162                     }
1163                 }
1164             }
1165             break
1166         case 'r2r_jitstress1':
1167         case 'r2r_jitstress2':
1168         case 'r2r_jitstressregs1':
1169         case 'r2r_jitstressregs2':
1170         case 'r2r_jitstressregs3':
1171         case 'r2r_jitstressregs4':
1172         case 'r2r_jitstressregs8':
1173         case 'r2r_jitstressregs0x10':
1174         case 'r2r_jitstressregs0x80':
1175         case 'r2r_jitstressregs0x1000':
1176         case 'r2r_jitminopts':
1177         case 'r2r_jitforcerelocs':
1178         case 'r2r_gcstress15':
1179             assert !(os in bidailyCrossList)
1180
1181             // GCStress=C is currently not supported on OS X
1182             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1183                 break
1184             }
1185
1186             // GC Stress 15 r2r gets a push trigger for checked/release
1187             if (configuration == 'Checked' || configuration == 'Release') {
1188                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1189                 if (architecture == 'x64') {
1190                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1191                     if (isFlowJob || os == 'Windows_NT') {
1192                         // Add a weekly periodic trigger
1193                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1194                     }
1195                 }
1196                 // For x86, only add per-commit jobs for Windows
1197                 else if (architecture == 'x86') {
1198                     if (os == 'Windows_NT') {
1199                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1200                     }
1201                 }
1202             }
1203             break
1204         case 'longgc':
1205             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1206             assert configuration == 'Release'
1207             assert architecture == 'x64'
1208             addPeriodicTriggerHelper(job, '@daily')
1209             // TODO: Add once external email sending is available again
1210             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1211             break
1212         case 'gcsimulator':
1213             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1214             assert configuration == 'Release'
1215             assert architecture == 'x64'
1216             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1217             // TODO: Add once external email sending is available again
1218             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1219             break
1220         case 'standalone_gc':
1221             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1222             assert (configuration == 'Release' || configuration == 'Checked')
1223             // TODO: Add once external email sending is available again
1224             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1225             addPeriodicTriggerHelper(job, '@daily')
1226             break
1227         case 'gc_reliability_framework':
1228             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1229             assert (configuration == 'Release' || configuration == 'Checked')
1230             // Only triggered by phrase.
1231             break
1232         case 'ilrt':
1233             assert !(os in bidailyCrossList)
1234             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1235             if (architecture == 'x64' && configuration == 'Release') {
1236                 // We don't expect to see a job generated except in these scenarios
1237                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1238                 if (isFlowJob || os == 'Windows_NT') {
1239                     addPeriodicTriggerHelper(job, '@daily')
1240                 }
1241             }
1242             break
1243         case 'jitdiff':
1244             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1245             assert configuration == 'Checked'
1246             assert (architecture == 'x64' || architecture == 'x86')
1247             addGithubPushTriggerHelper(job)
1248             break
1249         case 'formatting':
1250             assert (os == 'Windows_NT' || os == "Ubuntu")
1251             assert architecture == 'x64'
1252             addGithubPushTriggerHelper(job)
1253             break
1254         case 'jitstressregs1':
1255         case 'jitstressregs2':
1256         case 'jitstressregs3':
1257         case 'jitstressregs4':
1258         case 'jitstressregs8':
1259         case 'jitstressregs0x10':
1260         case 'jitstressregs0x80':
1261         case 'jitstressregs0x1000':
1262         case 'minopts':
1263         case 'forcerelocs':
1264         case 'jitstress1':
1265         case 'jitstress2':
1266         case 'jitstress2_jitstressregs1':
1267         case 'jitstress2_jitstressregs2':
1268         case 'jitstress2_jitstressregs3':
1269         case 'jitstress2_jitstressregs4':
1270         case 'jitstress2_jitstressregs8':
1271         case 'jitstress2_jitstressregs0x10':
1272         case 'jitstress2_jitstressregs0x80':
1273         case 'jitstress2_jitstressregs0x1000':
1274         case 'tailcallstress':
1275         case 'jitsse2only':
1276         case 'jitnosimd':
1277         case 'jitnox86hwintrinsic':
1278         case 'jitincompletehwintrinsic':
1279         case 'jitx86hwintrinsicnoavx':
1280         case 'jitx86hwintrinsicnoavx2':
1281         case 'jitx86hwintrinsicnosimd':
1282         case 'corefx_baseline':
1283         case 'corefx_minopts':
1284         case 'corefx_jitstress1':
1285         case 'corefx_jitstress2':
1286         case 'corefx_jitstressregs1':
1287         case 'corefx_jitstressregs2':
1288         case 'corefx_jitstressregs3':
1289         case 'corefx_jitstressregs4':
1290         case 'corefx_jitstressregs8':
1291         case 'corefx_jitstressregs0x10':
1292         case 'corefx_jitstressregs0x80':
1293         case 'corefx_jitstressregs0x1000':
1294         case 'zapdisable':
1295             if (os == 'CentOS7.1') {
1296                 break
1297             }
1298             if (os in bidailyCrossList) {
1299                 break
1300             }
1301             assert (os == 'Windows_NT') || (os in Constants.crossList)
1302             if (jobRequiresLimitedHardware(architecture, os)) {
1303                 addPeriodicTriggerHelper(job, '@weekly')
1304             }
1305             else {
1306                 addPeriodicTriggerHelper(job, '@daily')
1307             }
1308             break
1309         case 'heapverify1':
1310         case 'gcstress0x3':
1311             if (os == 'CentOS7.1') {
1312                 break
1313             }
1314             if (os in bidailyCrossList) {
1315                 break
1316             }
1317             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1318                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1319                 break
1320             }
1321             assert (os == 'Windows_NT') || (os in Constants.crossList)
1322             addPeriodicTriggerHelper(job, '@weekly')
1323             break
1324         case 'gcstress0xc':
1325         case 'gcstress0xc_zapdisable':
1326         case 'gcstress0xc_zapdisable_jitstress2':
1327         case 'gcstress0xc_zapdisable_heapverify1':
1328         case 'gcstress0xc_jitstress1':
1329         case 'gcstress0xc_jitstress2':
1330         case 'gcstress0xc_minopts_heapverify1':
1331             if (os == 'CentOS7.1') {
1332                 break
1333             }
1334             if (os == 'OSX10.12') {
1335                 // GCStress=C is currently not supported on OS X
1336                 break
1337             }
1338             if (os in bidailyCrossList) {
1339                 break
1340             }
1341             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1342                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1343                 break
1344             }
1345             assert (os == 'Windows_NT') || (os in Constants.crossList)
1346             addPeriodicTriggerHelper(job, '@weekly')
1347             break
1348
1349         case 'illink':
1350             // Testing on other operating systems TBD
1351             assert (os == 'Windows_NT' || os == 'Ubuntu')
1352             if (architecture == 'x64' || architecture == 'x86') {
1353                 if (configuration == 'Checked') {
1354                     addPeriodicTriggerHelper(job, '@daily')
1355                 }
1356             }
1357             break
1358
1359         case 'tieredcompilation':
1360         case 'corefx_tieredcompilation':
1361             // No periodic jobs just yet, still testing
1362             break
1363
1364         default:
1365             println("Unknown scenario: ${scenario}");
1366             assert false
1367             break
1368     }
1369     return
1370 }
1371
1372 // **************************
1373 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1374 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1375 // builds will do a couple extra steps.
1376 // **************************
1377
1378 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1379 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1380 // it's just used for internal builds
1381 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1382 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1383     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1384
1385     if (isWindowsBuildOnlyJob) {
1386         return
1387     }
1388
1389     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1390     // Non pull request builds.
1391     if (!isPR) {
1392         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1393         return
1394     }
1395
1396      def arm64Users = [
1397         'AndyAyersMS',
1398         'briansull',
1399         'BruceForstall',
1400         'CarolEidt',
1401         'cmckinsey',
1402         'echesakovMSFT',
1403         'erozenfeld',
1404         'janvorli',
1405         'jashook',
1406         'JosephTremoulet',
1407         'pgodeq',
1408         'russellhadley',
1409         'RussKeldorph',
1410         'sandreenko',
1411         'sdmaclea',
1412         'swaroop-sridhar',
1413         'jkotas',
1414         'markwilkie',
1415         'weshaggard'
1416     ]
1417
1418     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1419     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1420     def osGroup = getOSGroup(os)
1421     switch (architecture) {
1422         case 'x64': // editor brace matching: {
1423             if (scenario == 'formatting') {
1424                 assert configuration == 'Checked'
1425                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1426                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1427                 }
1428
1429                 break
1430             }
1431
1432             switch (os) {
1433                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1434                 case 'Debian8.4':
1435                 case 'RHEL7.2':
1436                     if (scenario == 'innerloop') {
1437                         assert !isFlowJob
1438                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1439                     }
1440                     else if (scenario == 'normal') {
1441                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1442                     }
1443                     break
1444
1445                 case 'Ubuntu16.04':
1446                     assert !isFlowJob
1447                     assert scenario != 'innerloop'
1448                     // Distinguish with the other architectures (arm and x86)
1449                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1450                     break
1451
1452                 case 'Fedora24':
1453                 case 'Ubuntu16.10':
1454                     assert !isFlowJob
1455                     assert scenario != 'innerloop'
1456                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1457                     break
1458
1459                 case 'Ubuntu':
1460                     if (scenario == 'illink') {
1461                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1462                         break
1463                     }
1464                     // fall through
1465
1466                 case 'OSX10.12':
1467                     // Triggers on the non-flow jobs aren't necessary here
1468                     // Corefx testing uses non-flow jobs.
1469                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1470                         break
1471                     }
1472                     switch (scenario) {
1473                         case 'innerloop':
1474                             // PR Triggered jobs. These jobs will run pri0 tests.
1475                             if (configuration == 'Checked') {
1476                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1477                             }
1478                             break
1479
1480                         case 'normal':
1481                             // OSX uses checked for default PR tests
1482                             if (configuration == 'Checked') {
1483                                 // Default trigger
1484                                 assert !job.name.contains("centos")
1485                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1486                             }
1487                             break
1488
1489                         case 'jitdiff':
1490                             if (configuration == 'Checked') {
1491                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1492                             }
1493                             break
1494
1495                         case 'ilrt':
1496                             if (configuration == 'Release') {
1497                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1498                             }
1499                             break
1500
1501                         case 'longgc':
1502                             if (configuration == 'Release') {
1503                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1504                             }
1505                             break
1506
1507                         case 'gcsimulator':
1508                             if (configuration == 'Release') {
1509                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1510                             }
1511                             break
1512
1513                         case 'standalone_gc':
1514                             if (configuration == 'Release' || configuration == 'Checked') {
1515                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1516                             }
1517                             break
1518
1519                         case 'gc_reliability_framework':
1520                             if (configuration == 'Release' || configuration == 'Checked') {
1521                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1522                             }
1523                             break
1524
1525                         default:
1526                             if (isJitStressScenario(scenario)) {
1527                                 def displayStr = getStressModeDisplayName(scenario)
1528                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1529                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1530                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1531                             }
1532                             else if (isR2RScenario(scenario)) {
1533                                 if (configuration == 'Release' || configuration == 'Checked') {
1534                                     def displayStr = getR2RDisplayName(scenario)
1535                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1536                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1537                                 }
1538                             }
1539                             else {
1540                                 println("Unknown scenario: ${scenario}");
1541                                 assert false
1542                             }
1543                             break
1544
1545                     }
1546                     break
1547
1548                 case 'CentOS7.1':
1549                     switch (scenario) {
1550                         case 'innerloop':
1551                             // CentOS uses checked for default PR tests while debug is build only
1552                             if (configuration == 'Debug') {
1553                                 // Default trigger
1554                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1555                             }
1556
1557                             // Make sure this is a flow job to get build and test.
1558                             if (configuration == 'Checked' && isFlowJob) {
1559                                 assert job.name.contains("flow")
1560                                 // Default trigger
1561                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1562                             }
1563                             break
1564
1565                         case 'normal':
1566                             // Make sure this is a flow job to get build and test.
1567                             if (configuration == 'Checked' && isFlowJob) {
1568                                 assert job.name.contains("flow")
1569                                 // Default trigger
1570                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1571                             }
1572                             break
1573
1574                         default:
1575                             if (isR2RScenario(scenario)) {
1576                                 if (configuration == 'Release' || configuration == 'Checked') {
1577                                     def displayStr = getR2RDisplayName(scenario)
1578                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1579                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1580                                 }
1581                             }
1582                             break
1583
1584                     }
1585                     break
1586
1587                 case 'Windows_NT':
1588                     switch (scenario) {
1589                         case 'innerloop':
1590                             // Default trigger
1591                             if (configuration == 'Checked' || configuration == 'Release') {
1592                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1593                             }
1594                             break
1595
1596                         case 'normal':
1597                             if (configuration == 'Checked') {
1598                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1599                             }
1600                             break
1601
1602                         case 'jitdiff':
1603                             if (configuration == 'Checked') {
1604                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1605                             }
1606                             break
1607
1608                         case 'ilrt':
1609                             if (configuration == 'Release') {
1610                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1611                             }
1612                             break
1613
1614                         case 'longgc':
1615                             if (configuration == 'Release') {
1616                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1617                             }
1618                             break
1619
1620                         case 'gcsimulator':
1621                             if (configuration == 'Release') {
1622                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1623                             }
1624                             break
1625
1626                         case 'standalone_gc':
1627                             if (configuration == 'Release' || configuration == 'Checked') {
1628                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1629                             }
1630                             break
1631
1632                         case 'gc_reliability_framework':
1633                             if (configuration == 'Release' || configuration == 'Checked') {
1634                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1635                             }
1636                             break
1637
1638                         case 'illink':
1639                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1640                             break
1641
1642                         default:
1643                             if (isJitStressScenario(scenario)) {
1644                                 def displayStr = getStressModeDisplayName(scenario)
1645                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1646                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1647                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1648                             }
1649                             else if (isR2RScenario(scenario)) {
1650                                 if (configuration == 'Release' || configuration == 'Checked') {
1651                                     def displayStr = getR2RDisplayName(scenario)
1652                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1653                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1654                                 }
1655                             }
1656                             else {
1657                                 println("Unknown scenario: ${scenario}");
1658                                 assert false
1659                             }
1660                             break
1661
1662                     }
1663                     break
1664
1665                 default:
1666                     println("Unknown os: ${os}");
1667                     assert false
1668                     break
1669
1670             }
1671
1672             break
1673         // editor brace matching: }
1674
1675         case 'armem': // editor brace matching: {
1676             job.with {
1677                 publishers {
1678                     azureVMAgentPostBuildAction {
1679                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1680                     }
1681                 }
1682             }
1683
1684             switch (os) {
1685                 case 'Ubuntu':
1686                 case 'Ubuntu16.04':
1687                     assert scenario != 'innerloop'
1688                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1689                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1690                     break
1691
1692                 case 'Tizen':
1693                     architecture = 'armel'
1694
1695                     assert scenario != 'innerloop'
1696                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1697                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1698                     break
1699             }
1700
1701             break
1702         // editor brace matching: }
1703
1704         case 'armlb':
1705         case 'arm': // editor brace matching: {
1706
1707             // Triggers on the non-flow jobs aren't necessary
1708             if (!isFlowJob) {
1709                 break
1710             }
1711
1712             // Set up a private trigger
1713             def contextString = "${os} ${architecture} Cross ${configuration}"
1714             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1715             if (scenario == 'innerloop') {
1716                 contextString += " Innerloop"
1717                 triggerString += "\\W+Innerloop"
1718             }
1719             else {
1720                 contextString += " ${scenario}"
1721                 triggerString += "\\W+${scenario}"
1722             }
1723
1724             if (configuration == 'Debug') {
1725                 contextString += " Build"
1726                 triggerString += "\\W+Build"
1727             } else {
1728                 contextString += " Build and Test"
1729                 triggerString += "\\W+Build and Test"
1730             }
1731
1732             triggerString += ".*"
1733
1734             switch (os) {
1735                 case 'Ubuntu':
1736                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1737                         break
1738                     }
1739
1740                     if (scenario == 'innerloop') {
1741                         if (configuration == 'Checked') {
1742                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1743                         }
1744                     }
1745                     else {
1746                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1747                     }
1748                     break
1749
1750                 case 'Windows_NT':
1751                     if (architecture == "armlb") {
1752                         // Disable armlb windows jobs
1753                         break
1754                     }
1755                     switch (scenario) {
1756                         case 'innerloop':
1757                             // Only Checked is an innerloop trigger.
1758                             if (configuration == 'Checked')
1759                             {
1760                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1761                             }
1762                             break
1763                         case 'normal':
1764                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1765                             break
1766                         default:
1767                             // Stress jobs will use this code path.
1768                             if (isArmWindowsScenario(scenario)) {
1769                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1770                             }
1771                             break
1772                     }
1773                     break
1774                 default:
1775                     println("NYI os: ${os}");
1776                     assert false
1777                     break
1778             }
1779             break
1780         // editor brace matching: }
1781         case 'arm64': // editor brace matching: {
1782             // Set up a private trigger
1783             def contextString = "${os} ${architecture} Cross ${configuration}"
1784             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1785
1786             if (scenario == 'innerloop') {
1787                 contextString += " Innerloop"
1788                 triggerString += "\\W+Innerloop"
1789             }
1790             else {
1791                 contextString += " ${scenario}"
1792                 triggerString += "\\W+${scenario}"
1793             }
1794
1795             if (configuration == 'Debug') {
1796                 contextString += " Build"
1797                 triggerString += "\\W+Build"
1798             } else {
1799                 contextString += " Build and Test"
1800                 triggerString += "\\W+Build and Test"
1801             }
1802
1803             triggerString += ".*"
1804
1805             switch (os) {
1806                 case 'Ubuntu':
1807                 case 'Ubuntu16.04':
1808                     switch (scenario) {
1809                         case 'innerloop':
1810                             if (configuration == 'Debug' && !isFlowJob) {
1811                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1812                             }
1813
1814                             break
1815                         case 'normal':
1816                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1817                             break
1818                         default:
1819                             if (isR2RScenario(scenario)) {
1820                                 if (configuration == 'Checked' || configuration == 'Release') {
1821                                     def displayStr = getR2RDisplayName(scenario)
1822                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1823                                 }
1824                             }
1825                             break
1826                     }
1827                     break
1828
1829                 case 'Windows_NT':
1830                     // Triggers on the non-flow jobs aren't necessary here
1831                     if (!isFlowJob) {
1832                         break
1833                     }
1834
1835                     assert isArmWindowsScenario(scenario)
1836                     switch (scenario) {
1837                         case 'innerloop':
1838                             if (configuration == 'Checked') {
1839                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1840                             }
1841
1842                             break
1843                         case 'normal':
1844                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1845                             break
1846                         default:
1847                             // Stress jobs will use this code path.
1848                             if (isArmWindowsScenario(scenario)) {
1849                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1850                             }
1851                             break
1852                     }
1853                     break
1854                 default:
1855                     println("NYI os: ${os}");
1856                     assert false
1857                     break
1858             }
1859             break
1860
1861         // editor brace matching: }
1862         case 'x86': // editor brace matching: {
1863             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1864             if (os == 'Ubuntu') {
1865                 // Triggers on the non-flow jobs aren't necessary here
1866                 if (!isFlowJob) {
1867                     break
1868                 }
1869
1870                 // on-demand only for ubuntu x86
1871                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1872                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1873                 break
1874
1875             }
1876             switch (scenario) {
1877                 case 'innerloop':
1878                     if (configuration == 'Checked' || configuration == 'Release') {
1879                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1880                     }
1881                     break
1882
1883                 case 'normal':
1884                     if (configuration == 'Checked') {
1885                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1886                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1887                     }
1888                     break
1889
1890                 case 'ilrt':
1891                     if (configuration == 'Release') {
1892                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1893                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1894                     }
1895                     break
1896
1897                 case 'longgc':
1898                     if (configuration == 'Release') {
1899                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1900                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1901                     }
1902                     break
1903
1904                 case 'gcsimulator':
1905                     if (configuration == 'Release') {
1906                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1907                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1908                     }
1909                     break
1910
1911                 case 'standalone_gc':
1912                     if (configuration == 'Release' || configuration == 'Checked') {
1913                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1914                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1915                     }
1916                     break
1917
1918                 case 'illink':
1919                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1920                     break
1921
1922                 default:
1923                     if (isJitStressScenario(scenario)) {
1924                         def displayStr = getStressModeDisplayName(scenario)
1925                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1926                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1927                     }
1928                     else if (isR2RScenario(scenario)) {
1929                         if (configuration == 'Release' || configuration == 'Checked') {
1930                             def displayStr = getR2RDisplayName(scenario)
1931                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1932                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1933                         }
1934                     }
1935                     else {
1936                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1937                         assert false
1938                     }
1939                     break
1940
1941             }
1942             break
1943
1944          // editor brace matching: }
1945         case 'x64_arm64_altjit':
1946         case 'x86_arm_altjit': // editor brace matching: {
1947             assert (os == 'Windows_NT')
1948             switch (scenario) {
1949                 case 'normal':
1950                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1951                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1952                     break
1953                 default:
1954                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1955                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1956                     break
1957             }
1958             break
1959
1960         // editor brace matching: }
1961         default:
1962             println("Unknown architecture: ${architecture}");
1963             assert false
1964             break
1965     }
1966 }
1967
1968 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1969     def buildCommands = []
1970     def osGroup = getOSGroup(os)
1971     def lowerConfiguration = configuration.toLowerCase()
1972
1973     def priority = '1'
1974     if (scenario == 'innerloop') {
1975         priority = '0'
1976     }
1977
1978     def doCoreFxTesting = isCoreFxScenario(scenario)
1979
1980     // Calculate the build steps, archival, and xunit results
1981     switch (os) {
1982         case 'Windows_NT': // editor brace matching: {
1983             switch (architecture) {
1984                 case 'x64':
1985                 case 'x86':
1986                 case 'x86_arm_altjit':
1987                 case 'x64_arm64_altjit':
1988                     def arch = architecture
1989                     def buildOpts = ''
1990                     if (architecture == 'x86_arm_altjit') {
1991                         arch = 'x86'
1992                     }
1993                     else if (architecture == 'x64_arm64_altjit') {
1994                         arch = 'x64'
1995                     }
1996
1997                     if (scenario == 'formatting') {
1998                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1999                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2000                         break
2001                     }
2002
2003                     if (scenario == 'illink') {
2004                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2005                     }
2006
2007                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2008                     if ((lowerConfiguration == 'release') &&
2009                         (scenario in Constants.basicScenarios) &&
2010                         (architecture != 'x86_arm_altjit') &&
2011                         (architecture != 'x64_arm64_altjit')) {
2012
2013                         buildOpts += ' -enforcepgo'
2014                     }
2015
2016                     if (doCoreFxTesting) {
2017                         buildOpts += ' skiptests';
2018                     } else {
2019                         buildOpts += " -priority=${priority}"
2020                     }
2021
2022                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2023                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2024                     // 35 characters long.
2025
2026                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2027
2028                     if (!isBuildOnly) {
2029                         def runtestArguments = ''
2030                         def testOpts = 'collectdumps'
2031
2032                         if (isR2RScenario(scenario)) {
2033
2034                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2035                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2036                             // to cause the tests to be crossgen'ed.
2037
2038                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2039                                 testOpts += ' crossgenaltjit protononjit.dll'
2040                             } else {
2041                                 testOpts += ' crossgen'
2042                             }
2043
2044                             testOpts += ' runcrossgentests'
2045                         }
2046                         else if (scenario == 'jitdiff') {
2047                             testOpts += ' jitdisasm crossgen'
2048                         }
2049                         else if (scenario == 'ilrt') {
2050                             testOpts += ' ilasmroundtrip'
2051                         }
2052                         else if (isLongGc(scenario)) {
2053                             testOpts += " ${scenario} sequential"
2054                         }
2055                         else if (scenario == 'standalone_gc') {
2056                             testOpts += ' gcname clrgc.dll'
2057                         }
2058                         else if (scenario == 'illink') {
2059                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2060                         }
2061
2062                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2063                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2064                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2065                         // which is read by the xunit harness.)
2066                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2067                         {
2068                             def timeout = 1800000
2069                             testOpts += " timeout ${timeout}"
2070                         }
2071
2072                         // If we are running a stress mode, we should write out the set of key
2073                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2074
2075                         def envScriptPath = ''
2076                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2077                             def buildCommandsStr = ''
2078                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2079                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2080
2081                             if (isJitStressScenario(scenario)) {
2082                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2083                             }
2084                             else if (isR2RStressScenario(scenario)) {
2085                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2086                             }
2087
2088                             if (architecture == 'x86_arm_altjit') {
2089                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2090                             }
2091                             else if (architecture == 'x64_arm64_altjit') {
2092                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2093                             }
2094
2095                             envScriptFinalize(os, envScriptPath)
2096
2097                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2098                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2099                             // into a single script.
2100                             buildCommands += buildCommandsStr
2101                         }
2102                         else if (architecture == 'x86_arm_altjit') {
2103                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2104                         }
2105                         else if (architecture == 'x64_arm64_altjit') {
2106                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2107                         }
2108                         if (envScriptPath != '') {
2109                             testOpts += " TestEnv ${envScriptPath}"
2110                         }
2111
2112                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2113
2114                         if (doCoreFxTesting) {
2115                             def workspaceRelativeFxRoot = "_/fx"
2116                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2117
2118                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2119
2120                             // Archive and process (only) the test results
2121                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2122                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2123
2124                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2125                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2126                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2127                         }
2128                         else if (isGcReliabilityFramework(scenario)) {
2129                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2130                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2131                         }
2132                         else {
2133                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2134                         }
2135                     } // end if (!isBuildOnly)
2136
2137                     if (!doCoreFxTesting) {
2138                         // Run the rest of the build
2139                         // Build the mscorlib for the other OS's
2140                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2141                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2142
2143                         if (arch == 'x64') {
2144                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2145                         }
2146
2147                         // Zip up the tests directory so that we don't use so much space/time copying
2148                         // 10s of thousands of files around.
2149                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2150
2151                         if (!isJitStressScenario(scenario)) {
2152                             // For Windows, pull full test results and test drops for x86/x64.
2153                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2154                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2155                         }
2156
2157                         if (scenario == 'jitdiff') {
2158                             // retrieve jit-dasm output for base commit, and run jit-diff
2159                             if (!isBuildOnly) {
2160                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2161                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2162                             }
2163                         }
2164
2165                         if (!isBuildOnly) {
2166                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2167                         }
2168                     }
2169                     break
2170                 case 'armlb':
2171                 case 'arm':
2172                     assert isArmWindowsScenario(scenario)
2173
2174                     def buildArchitecture = 'arm'
2175
2176                     def buildOpts = ''
2177
2178                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2179                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2180                     if (architecture == 'armlb') {
2181                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2182                     }
2183
2184                     if (doCoreFxTesting) {
2185                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2186                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2187                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2188                         //
2189                         // buildOpts += ' skiptests';
2190                         buildOpts += " -priority=0"
2191                     } else {
2192                         buildOpts += " -priority=${priority}"
2193                     }
2194
2195                     // This is now a build only job. Do not run tests. Use the flow job.
2196                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2197
2198                     if (doCoreFxTesting) {
2199                         assert isBuildOnly
2200                         assert architecture == 'arm'
2201
2202                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2203                         // to determine the correct matching corefx version git commit hash.
2204                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2205
2206                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2207                         def envScriptPath = ''
2208                         def buildCommandsStr = ''
2209                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2210                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2211                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2212                         envScriptFinalize(os, envScriptPath)
2213                         buildCommands += buildCommandsStr
2214
2215                         def workspaceRelativeFxRootLinux = "_/fx"
2216                         def workspaceRelativeFxRootWin = "_\\fx"
2217                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2218
2219                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2220
2221                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2222                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2223                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2224
2225                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2226                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2227                     } else {
2228                         // Zip up the tests directory so that we don't use so much space/time copying
2229                         // 10s of thousands of files around.
2230                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2231
2232                         // Add archival.
2233                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2234                     }
2235                     break
2236                 case 'arm64':
2237                     assert isArmWindowsScenario(scenario)
2238
2239                     // This is now a build only job. Do not run tests. Use the flow job.
2240                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2241
2242                     // Zip up the tests directory so that we don't use so much space/time copying
2243                     // 10s of thousands of files around.
2244                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2245
2246                     // Add archival.
2247                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2248                     break
2249                 default:
2250                     println("Unknown architecture: ${architecture}");
2251                     assert false
2252                     break
2253             }
2254             break
2255         // end case 'Windows_NT'; editor brace matching: }
2256         case 'Ubuntu':
2257         case 'Ubuntu16.04':
2258         case 'Ubuntu16.10':
2259         case 'Debian8.4':
2260         case 'OSX10.12':
2261         case 'CentOS7.1':
2262         case 'RHEL7.2':
2263         case 'Tizen':
2264         case 'Fedora24': // editor brace matching: {
2265             switch (architecture) {
2266                 case 'x64':
2267                 case 'x86':
2268                     if (architecture == 'x86' && os == 'Ubuntu') {
2269                         // build and PAL test
2270                         def dockerImage = getDockerImageName(architecture, os, true)
2271                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2272                         dockerImage = getDockerImageName(architecture, os, false)
2273                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2274                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2275                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2276                         break
2277                     }
2278
2279                     if (scenario == 'formatting') {
2280                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2281                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2282                         break
2283                     }
2284
2285                     if (scenario == 'illink') {
2286                         assert(os == 'Ubuntu')
2287                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2288                     }
2289
2290                     if (!doCoreFxTesting) {
2291                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2292                         // only on supported OS platforms.
2293                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2294                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2295
2296                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2297                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2298
2299                         // Basic archiving of the build
2300                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2301                         // And pal tests
2302                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2303                     }
2304                     else {
2305                         // Corefx stress testing
2306                         assert os == 'Ubuntu'
2307                         assert architecture == 'x64'
2308                         assert lowerConfiguration == 'checked'
2309                         assert isJitStressScenario(scenario)
2310
2311                         // Build coreclr
2312                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2313
2314                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2315
2316                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2317                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2318                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2319                         buildCommands += envScriptCmds
2320
2321                         // Build and text corefx
2322                         def workspaceRelativeFxRoot = "_/fx"
2323                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2324
2325                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2326
2327                         // Archive and process (only) the test results
2328                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2329                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2330                     }
2331                     break
2332                 case 'arm64':
2333                     if (!doCoreFxTesting) {
2334                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2335
2336                         // HACK -- Arm64 does not have corefx jobs yet.
2337                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2338                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2339                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2340                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2341
2342                         // Basic archiving of the build
2343                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2344                     }
2345                     break
2346                 case 'armem':
2347                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2348                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2349
2350                     // default values for Ubuntu
2351                     def arm_abi = "arm"
2352                     def linuxCodeName = "trusty"
2353                     if (os == 'Ubuntu16.04') {
2354                         linuxCodeName = "xenial"
2355                     }
2356                     else if (os == 'Tizen') {
2357                         arm_abi = "armel"
2358                         linuxCodeName = "tizen"
2359                     }
2360
2361                     // Unzip the Windows test binaries first. Exit with 0
2362                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2363
2364                     // Unpack the corefx binaries
2365                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2366                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2367                     if (os != 'Tizen') {
2368                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2369                     }
2370                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2371                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2372                     if (os == 'Ubuntu') {
2373                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2374                     }
2375
2376                     // Call the ARM CI script to cross build and test using docker
2377                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2378                     --mode=docker \\
2379                     --${arm_abi} \\
2380                     --linuxCodeName=${linuxCodeName} \\
2381                     --buildConfig=${lowerConfiguration} \\
2382                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2383                     --coreFxBinDir=./bin/CoreFxBinDir \\
2384                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2385
2386                     // Basic archiving of the build, no pal tests
2387                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2388                     break
2389                 case 'arm':
2390                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2391                     assert (os == 'Ubuntu')
2392
2393                     // Add some useful information to the log file. Ignore return codes.
2394                     buildCommands += "uname -a || true"
2395
2396                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2397                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2398
2399                     def dockerImage = getDockerImageName(architecture, os, true)
2400                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm -e CAC_ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} "
2401
2402                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2403
2404                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2405                     // download the appropriate CoreFX packages.
2406                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2407                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2408
2409                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2410
2411                     // ZIP up for the test job (created in the flow job code):
2412                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2413                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2414                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2415                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2416
2417                     // These commands are assumed to be run from the root of the workspace.
2418                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2419                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2420
2421                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2422
2423                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2424                     // the cleanup task in Jenkins can't remove.
2425                     newJob.with {
2426                         publishers {
2427                             azureVMAgentPostBuildAction {
2428                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2429                             }
2430                         }
2431                     }
2432
2433                     break
2434                 default:
2435                     println("Unknown architecture: ${architecture}");
2436                     assert false
2437                     break
2438             }
2439             break
2440         // editor brace matching: }
2441         default:
2442             println("Unknown os: ${os}");
2443             assert false
2444             break
2445     } // os
2446
2447     return buildCommands
2448 }
2449
2450 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2451 // Returns true if the job should be generated.
2452 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2453 {
2454     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2455     // All other scenarios do Pri-1 testing.
2456     if (scenario == 'innerloop' && !isPR) {
2457         return false
2458     }
2459
2460     // Tizen is only supported for armem architecture
2461     if (os == 'Tizen' && architecture != 'armem') {
2462         return false
2463     }
2464
2465     // Filter based on architecture.
2466
2467     switch (architecture) {
2468         case 'arm64':
2469         case 'arm':
2470             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2471                 return false
2472             }
2473             break
2474         case 'armem':
2475             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2476                 return false
2477             }
2478             break
2479         case 'armlb':
2480             // Do not create armlb jobs
2481             return false
2482         case 'x86_arm_altjit':
2483         case 'x64_arm64_altjit':
2484             if (os != 'Windows_NT') {
2485                 return false
2486             }
2487             break
2488         case 'x86':
2489             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2490                 return false
2491             }
2492             break
2493         case 'x64':
2494             // Everything implemented
2495             break
2496         default:
2497             println("Unknown architecture: ${architecture}")
2498             assert false
2499             break
2500     }
2501
2502     // Which (Windows) build only jobs are required?
2503
2504     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2505
2506     if (isBuildOnly) {
2507         switch (architecture) {
2508             case 'arm':
2509                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2510                 if (!isCoreFxScenario(scenario)) {
2511                     return false
2512                 }
2513                 break
2514             case 'x64':
2515             case 'x86':
2516                 if (!isNormalOrInnerloop) {
2517                     return false
2518                 }
2519                 break
2520             default:
2521                 return false
2522         }
2523     }
2524
2525     // Filter based on scenario.
2526
2527     if (isJitStressScenario(scenario)) {
2528         if (configuration != 'Checked') {
2529             return false
2530         }
2531
2532         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2533         if (!isEnabledOS) {
2534             return false
2535         }
2536
2537         switch (architecture) {
2538             case 'x64':
2539             case 'x86_arm_altjit':
2540             case 'x64_arm64_altjit':
2541                 break
2542
2543             case 'x86':
2544                 // x86 ubuntu: no stress modes
2545                 if (os == 'Ubuntu') {
2546                     return false
2547                 }
2548                 break
2549
2550             case 'arm':
2551                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2552                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2553                     return false
2554                 }
2555                 break
2556
2557             default:
2558                 // arm64, armlb: stress is handled through flow jobs.
2559                 // armem: no stress jobs for ARM emulator.
2560                 return false
2561         }
2562     }
2563     else if (isR2RScenario(scenario)) {
2564         if (os != 'Windows_NT') {
2565             return false
2566         }
2567         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2568         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2569             return false
2570         }
2571     }
2572     else {
2573         // Skip scenarios
2574         switch (scenario) {
2575             case 'ilrt':
2576                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2577                 // the default scenario build
2578                 if (os != 'Windows_NT') {
2579                     return false
2580                 }
2581                 // Only x64 for now
2582                 if (architecture != 'x64') {
2583                     return false
2584                 }
2585                 // Release only
2586                 if (configuration != 'Release') {
2587                     return false
2588                 }
2589                 break
2590             case 'jitdiff':
2591                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2592                     return false
2593                 }
2594                 if (architecture != 'x64') {
2595                     return false
2596                 }
2597                 if (configuration != 'Checked') {
2598                     return false
2599                 }
2600                 break
2601             case 'longgc':
2602             case 'gcsimulator':
2603                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2604                     return false
2605                 }
2606                 if (architecture != 'x64') {
2607                     return false
2608                 }
2609                 if (configuration != 'Release') {
2610                     return false
2611                 }
2612                 break
2613             case 'gc_reliability_framework':
2614             case 'standalone_gc':
2615                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2616                     return false
2617                 }
2618
2619                 if (architecture != 'x64') {
2620                     return false
2621                 }
2622
2623                 if (configuration != 'Release' && configuration != 'Checked') {
2624                     return false
2625                 }
2626                 break
2627             // We only run Windows and Ubuntu x64 Checked for formatting right now
2628             case 'formatting':
2629                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2630                     return false
2631                 }
2632                 if (architecture != 'x64') {
2633                     return false
2634                 }
2635                 if (configuration != 'Checked') {
2636                     return false
2637                 }
2638                 break
2639             case 'illink':
2640                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2641                     return false
2642                 }
2643                 if (architecture != 'x64' && architecture != 'x86') {
2644                     return false
2645                 }
2646                 break
2647             case 'normal':
2648                 // Nothing skipped
2649                 break
2650             case 'innerloop':
2651                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2652                     return false
2653                 }
2654                 break
2655             default:
2656                 println("Unknown scenario: ${scenario}")
2657                 assert false
2658                 break
2659         }
2660     }
2661
2662     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2663     // runtime behavior, not compile-time behavior, are not interesting.
2664     switch (architecture) {
2665         case 'x86_arm_altjit':
2666         case 'x64_arm64_altjit':
2667             if (isGCStressRelatedTesting(scenario)) {
2668                 return false
2669             }
2670             break
2671         default:
2672             break
2673     }
2674
2675     // The job was not filtered out, so we should generate it!
2676     return true
2677 }
2678
2679 Constants.allScenarios.each { scenario ->
2680     [true, false].each { isPR ->
2681         Constants.architectureList.each { architecture ->
2682             Constants.configurationList.each { configuration ->
2683                 Constants.osList.each { os ->
2684                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2685                     // and reset the os to Windows_NT
2686                     def isBuildOnly = false
2687                     if (os == 'Windows_NT_BuildOnly') {
2688                         isBuildOnly = true
2689                         os = 'Windows_NT'
2690                     }
2691
2692                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2693                         return
2694                     }
2695
2696                     // Calculate names
2697                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2698                     def folderName = getJobFolder(scenario)
2699
2700                     // Create the new job
2701                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2702                     addToViews(newJob, isPR, architecture, os)
2703
2704                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2705
2706                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2707                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2708                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2709
2710                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2711                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2712                     // artifacts produced.
2713                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2714                         // Define the Windows Tests and Corefx build job names
2715                         def lowerConfiguration = configuration.toLowerCase()
2716                         def WindowsTestsName = projectFolder + '/' +
2717                                                Utilities.getFullJobName(project,
2718                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2719                                                                         false)
2720                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2721                                            Utilities.getFolderName(branch)
2722
2723                         def arm_abi = 'arm'
2724                         def corefx_os = 'linux'
2725                         if (os == 'Tizen') {
2726                             arm_abi = 'armel'
2727                             corefx_os = 'tizen'
2728                         }
2729
2730                         // Let's use release CoreFX to test checked CoreCLR,
2731                         // because we do not generate checked CoreFX in CoreFX CI yet.
2732                         def corefx_lowerConfiguration = lowerConfiguration
2733                         if (lowerConfiguration == 'checked') {
2734                             corefx_lowerConfiguration = 'release'
2735                         }
2736
2737                         // Copy the Windows test binaries and the Corefx build binaries
2738                         newJob.with {
2739                             steps {
2740                                 copyArtifacts(WindowsTestsName) {
2741                                     includePatterns('bin/tests/tests.zip')
2742                                     buildSelector {
2743                                         latestSuccessful(true)
2744                                     }
2745                                 }
2746                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2747                                     includePatterns('bin/build.tar.gz')
2748                                     buildSelector {
2749                                         latestSuccessful(true)
2750                                     }
2751                                 }
2752                             } // steps
2753                         } // newJob.with
2754                     }
2755
2756                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2757
2758                     newJob.with {
2759                         steps {
2760                             if (os == 'Windows_NT') {
2761                                 buildCommands.each { buildCommand ->
2762                                     batchFile(buildCommand)
2763                                 }
2764                             }
2765                             else {
2766                                 buildCommands.each { buildCommand ->
2767                                     shell(buildCommand)
2768                                 }
2769                             }
2770                         } // steps
2771                     } // newJob.with
2772
2773                 } // os
2774             } // configuration
2775         } // architecture
2776     } // isPR
2777 } // scenario
2778
2779 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2780 // Returns the newly created job.
2781 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2782 {
2783     def osGroup = getOSGroup(os)
2784     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2785
2786     def jobFolder = getJobFolder(scenario)
2787     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2788         parameters {
2789             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2790         }
2791
2792         steps {
2793             // Set up the copies
2794
2795             // Coreclr build we are trying to test
2796             //
2797             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2798
2799             copyArtifacts(inputCoreCLRBuildName) {
2800                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2801                 buildSelector {
2802                     buildNumber('${CORECLR_BUILD}')
2803                 }
2804             }
2805
2806             if (isCoreFxScenario(scenario)) {
2807
2808                 // Only arm supported for corefx testing now.
2809                 assert architecture == 'arm'
2810
2811                 // Unzip CoreFx runtime
2812                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2813
2814                 // Unzip CoreFx tests.
2815                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2816
2817                 // Add the script to run the corefx tests
2818                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2819                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2820                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2821                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2822
2823             } else { // !isCoreFxScenario(scenario)
2824
2825                 // Unzip tests.
2826                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2827
2828                 def buildCommands = ""
2829
2830                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2831                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2832                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2833
2834                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2835                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2836
2837                 // For all jobs
2838                 addEnvVariable("CORE_ROOT", coreRootLocation)
2839                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2840                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2841
2842                 // ARM legacy backend; this is an altjit.
2843                 if (architecture == 'armlb') {
2844                     addEnvVariable("COMPlus_AltJit", "*")
2845                     addEnvVariable("COMPlus_AltJitNgen", "*")
2846                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2847                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2848                 }
2849
2850                 // If we are running a stress mode, we'll set those variables as well
2851                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2852                     def stressValues = null
2853                     if (isJitStressScenario(scenario)) {
2854                         stressValues = Constants.jitStressModeScenarios[scenario]
2855                     }
2856                     else {
2857                         stressValues = Constants.r2rStressScenarios[scenario]
2858                     }
2859
2860                     stressValues.each { key, value ->
2861                         addEnvVariable(key, value)
2862                     }
2863                 }
2864
2865                 if (isR2RScenario(scenario)) {
2866                     // Crossgen the framework assemblies.
2867                     buildCommands += """
2868 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2869 @goto skip_PrecompileAssembly
2870
2871 :PrecompileAssembly
2872 @REM Skip mscorlib since it is already precompiled.
2873 @if /I "%3" == "mscorlib.dll" exit /b 0
2874 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2875
2876 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2877 @if "%errorlevel%" == "-2146230517" (
2878     echo %2 is not a managed assembly.
2879 ) else if "%errorlevel%" == "-2146234344" (
2880     echo %2 is not a managed assembly.
2881 ) else if %errorlevel% neq 0 (
2882     echo Unable to precompile %2
2883 ) else (
2884     echo Precompiled %2
2885 )
2886 @exit /b 0
2887
2888 :skip_PrecompileAssembly
2889 """
2890
2891                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2892                     // crossgen on tests before running them.
2893                     addEnvVariable("RunCrossGen", "true")
2894                 } // isR2RScenario(scenario)
2895
2896                 // Create the smarty command
2897                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2898                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2899                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2900                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2901
2902                 if (architecture == 'armlb') {
2903                     addExclude("LEGACYJIT_FAIL")
2904                 }
2905
2906                 // Exclude tests based on scenario.
2907                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2908                     addArchSpecificExclude(architecture, excludeTag)
2909                 }
2910
2911                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2912                 if (scenario == 'innerloop') {
2913                     addExclude("pri1")
2914                 }
2915
2916                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2917                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2918                 addExclude("LONG_RUNNING")
2919
2920                 smartyCommand += "/lstFile Tests.lst"
2921
2922                 def testListArch = [
2923                     'arm64': 'arm64',
2924                     'arm': 'arm',
2925                     'armlb': 'arm'
2926                 ]
2927
2928                 def archLocation = testListArch[architecture]
2929
2930                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2931                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2932                 addCommand("${smartyCommand}")
2933
2934                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2935                 // However, we also need to remove all the variables that were set during this batch file, so we
2936                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2937                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2938                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2939                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2940                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2941                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2942
2943                 // ZIP up the smarty output, no matter what the smarty result.
2944                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2945
2946                 addCommand("echo %errorlevel%")
2947                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2948
2949                 // Use the smarty errorlevel as the script errorlevel.
2950                 addCommand("exit /b %__save_smarty_errorlevel%")
2951
2952                 batchFile(buildCommands)
2953             } // non-corefx testing
2954         } // steps
2955     } // job
2956
2957     if (!isCoreFxScenario(scenario)) {
2958         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2959
2960         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2961         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2962         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2963         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2964     }
2965
2966     return newJob
2967 }
2968
2969 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2970 // E.g., non-Windows tests.
2971 // Returns the newly created job.
2972 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2973 {
2974     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2975
2976     def osGroup = getOSGroup(os)
2977     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2978
2979     def testOpts = ''
2980     def useServerGC = false
2981
2982     // Enable Server GC for Ubuntu PR builds
2983     // REVIEW: why? Does this apply to all architectures? Why only PR?
2984     if (os == 'Ubuntu' && isPR) {
2985         testOpts += ' --useServerGC'
2986         useServerGC = true
2987     }
2988
2989     if (isR2RScenario(scenario)) {
2990
2991         testOpts += ' --crossgen --runcrossgentests'
2992
2993         if (scenario == 'r2r_jitstress1') {
2994             testOpts += ' --jitstress=1'
2995         }
2996         else if (scenario == 'r2r_jitstress2') {
2997             testOpts += ' --jitstress=2'
2998         }
2999         else if (scenario == 'r2r_jitstressregs1') {
3000             testOpts += ' --jitstressregs=1'
3001         }
3002         else if (scenario == 'r2r_jitstressregs2') {
3003             testOpts += ' --jitstressregs=2'
3004         }
3005         else if (scenario == 'r2r_jitstressregs3') {
3006             testOpts += ' --jitstressregs=3'
3007         }
3008         else if (scenario == 'r2r_jitstressregs4') {
3009             testOpts += ' --jitstressregs=4'
3010         }
3011         else if (scenario == 'r2r_jitstressregs8') {
3012             testOpts += ' --jitstressregs=8'
3013         }
3014         else if (scenario == 'r2r_jitstressregs0x10') {
3015             testOpts += ' --jitstressregs=0x10'
3016         }
3017         else if (scenario == 'r2r_jitstressregs0x80') {
3018             testOpts += ' --jitstressregs=0x80'
3019         }
3020         else if (scenario == 'r2r_jitstressregs0x1000') {
3021             testOpts += ' --jitstressregs=0x1000'
3022         }
3023         else if (scenario == 'r2r_jitminopts') {
3024             testOpts += ' --jitminopts'
3025         }
3026         else if (scenario == 'r2r_jitforcerelocs') {
3027             testOpts += ' --jitforcerelocs'
3028         }
3029         else if (scenario == 'r2r_gcstress15') {
3030             testOpts += ' --gcstresslevel=0xF'
3031         }
3032     }
3033     else if (scenario == 'jitdiff') {
3034         testOpts += ' --jitdisasm --crossgen'
3035     }
3036     else if (scenario == 'illink') {
3037         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3038     }
3039     else if (isLongGc(scenario)) {
3040         // Long GC tests behave very poorly when they are not
3041         // the only test running (many of them allocate until OOM).
3042         testOpts += ' --sequential'
3043
3044         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3045         // for running long GC and GCSimulator tests, respectively. We don't use them
3046         // here because using a playlist file produces much more readable output on the CI machines
3047         // and reduces running time.
3048         //
3049         // The Long GC playlist contains all of the tests that are
3050         // going to be run. The GCSimulator playlist contains all of
3051         // the GC simulator tests.
3052         if (scenario == 'longgc') {
3053             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3054         }
3055         else if (scenario == 'gcsimulator') {
3056             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3057         }
3058     }
3059     else if (isGcReliabilityFramework(scenario)) {
3060         testOpts += ' --build-overlay-only'
3061     }
3062     else if (scenario == 'standalone_gc') {
3063         if (osGroup == 'OSX') {
3064             testOpts += ' --gcname=libclrgc.dylib'
3065         }
3066         else if (osGroup == 'Linux') {
3067             testOpts += ' --gcname=libclrgc.so'
3068         }
3069         else {
3070             println("Unexpected OS group: ${osGroup} for os ${os}")
3071             assert false
3072         }
3073     }
3074
3075     def jobFolder = getJobFolder(scenario)
3076     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3077         parameters {
3078             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3079             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3080         }
3081
3082         steps {
3083             // Set up the copies
3084
3085             // Coreclr build containing the tests and mscorlib
3086             // pri1 jobs still need to copy windows_nt built tests
3087             assert inputTestsBuildName != null
3088             copyArtifacts(inputTestsBuildName) {
3089                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3090                 buildSelector {
3091                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3092                 }
3093             }
3094
3095             // Coreclr build we are trying to test
3096             //
3097             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3098
3099             copyArtifacts(inputCoreCLRBuildName) {
3100                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3101                 buildSelector {
3102                     buildNumber('${CORECLR_BUILD}')
3103                 }
3104             }
3105
3106             if (isUbuntuArmJob) {
3107                 // Add some useful information to the log file. Ignore return codes.
3108                 shell("uname -a || true")
3109             }
3110
3111             if (architecture == 'arm64') {
3112                 shell("mkdir -p ./bin/CoreFxBinDir")
3113                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3114                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3115             }
3116             else if (architecture == 'x86') {
3117                 shell("mkdir ./bin/CoreFxNative")
3118
3119                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3120
3121                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3122                     includePatterns('bin/build.tar.gz')
3123                     targetDirectory('bin/CoreFxNative')
3124                     buildSelector {
3125                         latestSuccessful(true)
3126                     }
3127                 }
3128
3129                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3130             }
3131
3132             // Unzip the tests first.  Exit with 0
3133             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3134             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3135
3136             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3137             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3138             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3139             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3140             // copied correctly.
3141             if (isUbuntuArmJob) {
3142                 def lowerConfiguration = configuration.toLowerCase()
3143                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3144                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3145             }
3146             else {
3147                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3148             }
3149
3150             // Execute the tests
3151             def runDocker = isNeedDocker(architecture, os, false)
3152             def dockerPrefix = ""
3153             def dockerCmd = ""
3154             if (runDocker) {
3155                 def dockerImage = getDockerImageName(architecture, os, false)
3156                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3157                 dockerCmd = dockerPrefix + "${dockerImage} "
3158             }
3159
3160             // If we are running a stress mode, we'll set those variables first
3161             if (isJitStressScenario(scenario)) {
3162                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3163                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3164                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3165                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3166                 shell("${envScriptCmds}")
3167                 testOpts += " --test-env=${scriptFileName}"
3168             }
3169
3170             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3171             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3172             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3173             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3174             if (isGCStressRelatedTesting(scenario)) {
3175                 if (architecture == 'x64') {
3176                     shell('./init-tools.sh')
3177                 }
3178             }
3179
3180             def runScript = "${dockerCmd}./tests/runtest.sh"
3181
3182             shell("""\
3183 ${runScript} \\
3184     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3185     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3186     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3187     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3188
3189             if (isGcReliabilityFramework(scenario)) {
3190                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3191                 if (useServerGC) {
3192                     if (runDocker) {
3193                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3194                     }
3195                     else {
3196                         shell("export COMPlus_gcServer=1")
3197                     }
3198                 }
3199
3200                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3201             }
3202         } // steps
3203     } // job
3204
3205     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3206     if (os in ['Ubuntu']) {
3207         SummaryBuilder summaries = new SummaryBuilder()
3208         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3209         summaries.emit(newJob)
3210     }
3211
3212     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3213     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3214
3215     return newJob
3216 }
3217
3218 // Create a test job that will be used by a flow job.
3219 // Returns the newly created job.
3220 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3221 {
3222     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3223
3224     def newJob = null
3225     if (windowsArmJob) {
3226         assert inputTestsBuildName == null
3227         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3228     } else {
3229         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3230     }
3231
3232     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3233
3234     addToViews(newJob, isPR, architecture, os)
3235
3236     if (scenario == 'jitdiff') {
3237         def osGroup = getOSGroup(os)
3238         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3239     }
3240
3241     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3242     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3243
3244     return newJob
3245 }
3246
3247 // Create a flow job to tie together a build job with the given test job.
3248 // Returns the new flow job.
3249 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3250 {
3251     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3252     // Linux CoreCLR test
3253     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3254     def jobFolder = getJobFolder(scenario)
3255
3256     def newFlowJob = null
3257
3258     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3259     if (windowsArmJob) {
3260
3261         assert inputTestsBuildName == null
3262
3263         // For Windows arm jobs there is no reason to build a parallel test job.
3264         // The product build supports building and archiving the tests.
3265
3266         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3267                         buildFlow("""\
3268 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3269
3270 // And then build the test build
3271 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3272 """)
3273         }
3274         JobReport.Report.addReference(inputCoreCLRBuildName)
3275         JobReport.Report.addReference(fullTestJobName)
3276     }
3277     else {
3278         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3279                         buildFlow("""\
3280 // Build the input jobs in parallel
3281 parallel (
3282 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3283 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3284 )
3285
3286 // And then build the test build
3287 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3288                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3289 """)
3290         }
3291         JobReport.Report.addReference(inputCoreCLRBuildName)
3292         JobReport.Report.addReference(inputTestsBuildName)
3293         JobReport.Report.addReference(fullTestJobName)
3294     }
3295
3296     addToViews(newFlowJob, isPR, architecture, os)
3297
3298     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3299
3300     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3301     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3302
3303     return newFlowJob
3304 }
3305
3306 // Determine if we should generate a flow job for the given parameters.
3307 // Returns true if the job should be generated.
3308 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3309 {
3310     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3311     // All other scenarios do Pri-1 testing.
3312     if (scenario == 'innerloop' && !isPR) {
3313         return false
3314     }
3315
3316     // Filter based on OS and architecture.
3317
3318     switch (architecture) {
3319         case 'arm64':
3320             if (os != "Ubuntu" && os != "Windows_NT") {
3321                 return false
3322             }
3323             break
3324         case 'armlb':
3325             if (os != 'Windows_NT') {
3326                 return false
3327             }
3328             // Do not create armlb windows jobs.
3329             return false
3330         case 'arm':
3331             if (os != "Ubuntu" && os != "Windows_NT") {
3332                 return false
3333             }
3334             break
3335         case 'x86':
3336             if (os != "Ubuntu") {
3337                 return false
3338             }
3339             break
3340         case 'x64':
3341             if (!(os in Constants.crossList)) {
3342                 return false
3343             }
3344             if (os == "Windows_NT") {
3345                 return false
3346             }
3347             break
3348         case 'armem':
3349         case 'x86_arm_altjit':
3350         case 'x64_arm64_altjit':
3351             // No flow jobs
3352             return false
3353         default:
3354             println("Unknown architecture: ${architecture}")
3355             assert false
3356             break
3357     }
3358
3359     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3360
3361     // Filter based on scenario in OS.
3362
3363     if (os == 'Windows_NT') {
3364         if (!isArmWindowsScenario(scenario)) {
3365             return false
3366         }
3367     }
3368     else {
3369         // Non-Windows
3370         if (architecture == 'arm64') {
3371             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3372                 return false
3373             }
3374         }
3375         else if (architecture == 'arm') {
3376             if (!(scenario in Constants.validLinuxArmScenarios)) {
3377                 return false
3378             }
3379         }
3380         else if (architecture == 'x86') {
3381             // Linux/x86 only want innerloop and default test
3382             if (!isNormalOrInnerloop) {
3383                 return false
3384             }
3385         }
3386     }
3387
3388     // For CentOS, we only want Checked/Release builds.
3389     if (os == 'CentOS7.1') {
3390         if (configuration != 'Checked' && configuration != 'Release') {
3391             return false
3392         }
3393         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3394             return false
3395         }
3396     }
3397
3398     // For RedHat and Debian, we only do Release builds.
3399     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3400         if (configuration != 'Release') {
3401             return false
3402         }
3403         if (!isNormalOrInnerloop) {
3404             return false
3405         }
3406     }
3407
3408     // Next, filter based on scenario.
3409
3410     if (isJitStressScenario(scenario)) {
3411         if (configuration != 'Checked') {
3412             return false
3413         }
3414
3415         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3416         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3417             return false
3418         }
3419     }
3420     else if (isR2RBaselineScenario(scenario)) {
3421         if (configuration != 'Checked' && configuration != 'Release') {
3422             return false
3423         }
3424     }
3425     else if (isR2RStressScenario(scenario)) {
3426         if (configuration != 'Checked') {
3427             return false
3428         }
3429     }
3430     else {
3431         // Skip scenarios
3432         switch (scenario) {
3433             case 'ilrt':
3434             case 'longgc':
3435             case 'gcsimulator':
3436                 // Long GC tests take a long time on non-Release builds
3437                 // ilrt is also Release only
3438                 if (configuration != 'Release') {
3439                     return false
3440                 }
3441                 break
3442
3443             case 'jitdiff':
3444                 if (configuration != 'Checked') {
3445                     return false
3446                 }
3447                 break
3448
3449             case 'gc_reliability_framework':
3450             case 'standalone_gc':
3451                 if (configuration != 'Release' && configuration != 'Checked') {
3452                     return false
3453                 }
3454                 break
3455
3456             case 'formatting':
3457                 return false
3458             case 'illink':
3459                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3460                     return false
3461                 }
3462                 break
3463
3464             case 'normal':
3465                 // Nothing skipped
3466                 break
3467
3468             case 'innerloop':
3469                 // Nothing skipped
3470                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3471                     return false
3472                 }
3473                 break
3474
3475             default:
3476                 println("Unknown scenario: ${scenario}")
3477                 assert false
3478                 break
3479         }
3480     }
3481
3482     // The job was not filtered out, so we should generate it!
3483     return true
3484 }
3485
3486 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3487 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3488 Constants.allScenarios.each { scenario ->
3489     [true, false].each { isPR ->
3490         Constants.architectureList.each { architecture ->
3491             Constants.configurationList.each { configuration ->
3492                 Constants.osList.each { os ->
3493
3494                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3495                         return
3496                     }
3497
3498                     // Figure out the job name of the CoreCLR build the test will depend on.
3499
3500                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3501                     def inputCoreCLRBuildIsBuildOnly = false
3502                     if (isCoreFxScenario(scenario)) {
3503                         // Every CoreFx test depends on its own unique build.
3504                         inputCoreCLRBuildScenario = scenario
3505                         inputCoreCLRBuildIsBuildOnly = true
3506                     }
3507                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3508                     def inputCoreCLRBuildName = projectFolder + '/' +
3509                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3510
3511                     // Figure out the name of the build job that the test job will depend on.
3512                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3513                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3514
3515                     def inputTestsBuildName = null
3516
3517                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3518                     if (!windowsArmJob) {
3519                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3520
3521                         def inputTestsBuildArch = architecture
3522                         if (architecture == "arm64") {
3523                             // Use the x64 test build for arm64 unix
3524                             inputTestsBuildArch = "x64"
3525                         }
3526                         else if (architecture == "arm") {
3527                             // Use the x86 test build for arm unix
3528                             inputTestsBuildArch = "x86"
3529                         }
3530
3531                         def inputTestsBuildIsBuildOnly = true
3532
3533                         inputTestsBuildName = projectFolder + '/' +
3534                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3535                     }
3536
3537                     // =============================================================================================
3538                     // Create the test job
3539                     // =============================================================================================
3540
3541                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3542
3543                     // =============================================================================================
3544                     // Create a build flow to join together the build and tests required to run this test.
3545                     // =============================================================================================
3546
3547                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3548                         // Do not create the flow job for RHEL jobs.
3549                         return
3550                     }
3551
3552                     def fullTestJobName = projectFolder + '/' + testJob.name
3553                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3554
3555                 } // os
3556             } // configuration
3557         } // architecture
3558     } // isPR
3559 } // scenario
3560
3561 JobReport.Report.generateJobReport(out)
3562
3563 // Make the call to generate the help job
3564 Utilities.createHelperJob(this, project, branch,
3565     "Welcome to the ${project} Repository",  // This is prepended to the help message
3566     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3567
3568 Utilities.addCROSSCheck(this, project, branch)