netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213         'Tizen': [
 214             'armem': [
 215                 'Checked'
 216             ]
 217         ],
 218     ]
 219
 220     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 221     // to Tests.lst file categories to exclude.
 222     //
 223     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 224     // and with the same values, with some commented out, for easier maintenance.
 225     //
 226     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 227     //
 228     def static validArmWindowsScenarios = [
 229                'innerloop':                              [],
 230                'normal':                                 [],
 231                // 'ilrt'
 232                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 233                // 'longgc'
 234                // 'formatting'
 235                // 'gcsimulator'
 236                // 'jitdiff'
 237                // 'standalone_gc'
 238                // 'gc_reliability_framework'
 239                // 'illink'
 240                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 251                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 253                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 254                'tieredcompilation':                      [],
 255                'forcerelocs':                            [],
 256                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 275                // 'jitsse2only'                          // Only relevant to xarch
 276                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 277                // 'jitincompletehwintrinsic'
 278                // 'jitx86hwintrinsicnoavx'
 279                // 'jitx86hwintrinsicnoavx2'
 280                // 'jitx86hwintrinsicnosimd'
 281                // 'jitnox86hwintrinsic'
 282                'corefx_baseline':                        [],    // corefx tests don't use smarty
 283                'corefx_minopts':                         [],    // corefx tests don't use smarty
 284                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 285                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 286                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 290                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 291                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 292                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 293                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 294                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 295                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 296                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 297                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 298                'heapverify1':                            [],
 299                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 300                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 302                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 305
 306                //
 307                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 308                //
 309
 310                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 311                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 324                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 325                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 326                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 327                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 328                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 329     ]
 330
 331     def static validLinuxArm64Scenarios = [
 332                'innerloop',
 333                'normal',
 334                'r2r',
 335                'gcstress0x3',
 336                'gcstress0xc'
 337     ]
 338
 339     // Note: no GCStress-related scenario is enabled currently.
 340     def static validLinuxArmScenarios = [
 341                'innerloop',
 342                'normal',
 343                'r2r',
 344                'r2r_jitstress1',
 345                'r2r_jitstress2',
 346                'r2r_jitstressregs1',
 347                'r2r_jitstressregs2',
 348                'r2r_jitstressregs3',
 349                'r2r_jitstressregs4',
 350                'r2r_jitstressregs8',
 351                'r2r_jitstressregs0x10',
 352                'r2r_jitstressregs0x80',
 353                'r2r_jitstressregs0x1000',
 354                'r2r_jitminopts',
 355                'r2r_jitforcerelocs',
 356                // 'r2r_gcstress15',
 357                'minopts',
 358                'forcerelocs',
 359                'jitstress1',
 360                'jitstress2',
 361                'jitstressregs1',
 362                'jitstressregs2',
 363                'jitstressregs3',
 364                'jitstressregs4',
 365                'jitstressregs8',
 366                'jitstressregs0x10',
 367                'jitstressregs0x80',
 368                'jitstressregs0x1000',
 369                'jitstress2_jitstressregs1',
 370                'jitstress2_jitstressregs2',
 371                'jitstress2_jitstressregs3',
 372                'jitstress2_jitstressregs4',
 373                'jitstress2_jitstressregs8',
 374                'jitstress2_jitstressregs0x10',
 375                'jitstress2_jitstressregs0x80',
 376                'jitstress2_jitstressregs0x1000',
 377                'tailcallstress'
 378                // 'gcstress0x3',
 379                // 'gcstress0xc',
 380                // 'zapdisable',
 381                // 'heapverify1',
 382                // 'gcstress0xc_zapdisable',
 383                // 'gcstress0xc_zapdisable_jitstress2',
 384                // 'gcstress0xc_zapdisable_heapverify1',
 385                // 'gcstress0xc_jitstress1',
 386                // 'gcstress0xc_jitstress2',
 387                // 'gcstress0xc_minopts_heapverify1'
 388     ]
 389
 390     def static configurationList = ['Debug', 'Checked', 'Release']
 391
 392     // This is the set of architectures
 393     // Some of these are pseudo-architectures:
 394     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 395     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 396     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 397     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 398     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 399
 400     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 401     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 402 }
 403
 404 // **************************************************************
 405 // Create some specific views
 406 //
 407 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 408 // views based on a single regular expression. These views will be generated by adding a
 409 // specific set of jobs to them.
 410 //
 411 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 412 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 413 // those are required, don't add them (which simplifies the view pages, as well).
 414 // **************************************************************
 415
 416 class Views {
 417     def static MergeJobView = null
 418     def static PeriodicJobView = null
 419     def static ArchitectureViews = [:]
 420     def static OSViews = [:]
 421 }
 422
 423 // MergeJobView: include all jobs that execute when a PR change is merged.
 424 Views.MergeJobView = listView('Merge') {
 425     recurse()
 426     columns {
 427         status()
 428         weather()
 429         name()
 430         lastSuccess()
 431         lastFailure()
 432         lastDuration()
 433         buildButton()
 434     }
 435 }
 436
 437 // PeriodicJobView: include all jobs that execute on a schedule
 438 Views.PeriodicJobView = listView('Periodic') {
 439     recurse()
 440     columns {
 441         status()
 442         weather()
 443         name()
 444         lastSuccess()
 445         lastFailure()
 446         lastDuration()
 447         buildButton()
 448     }
 449 }
 450
 451 // Create a view for non-PR jobs for each architecture.
 452 Constants.architectureList.each { architecture ->
 453     Views.ArchitectureViews[architecture] = listView(architecture) {
 454         recurse()
 455         columns {
 456             status()
 457             weather()
 458             name()
 459             lastSuccess()
 460             lastFailure()
 461             lastDuration()
 462             buildButton()
 463         }
 464     }
 465 }
 466
 467 // Create a view for non-PR jobs for each OS.
 468 Constants.osList.each { os ->
 469     // Don't create one for the special 'Windows_NT_BuildOnly'
 470     if (os == 'Windows_NT_BuildOnly') {
 471         return
 472     }
 473     Views.OSViews[os] = listView(os) {
 474         recurse()
 475         columns {
 476             status()
 477             weather()
 478             name()
 479             lastSuccess()
 480             lastFailure()
 481             lastDuration()
 482             buildButton()
 483         }
 484     }
 485 }
 486
 487 def static addToMergeView(def job) {
 488     Views.MergeJobView.with {
 489         jobs {
 490             name(job.name)
 491         }
 492     }
 493 }
 494
 495 def static addToPeriodicView(def job) {
 496     Views.PeriodicJobView.with {
 497         jobs {
 498             name(job.name)
 499         }
 500     }
 501 }
 502
 503 def static addToViews(def job, def isPR, def architecture, def os) {
 504     if (isPR) {
 505         // No views want PR jobs currently.
 506         return
 507     }
 508
 509     // Add to architecture view.
 510     Views.ArchitectureViews[architecture].with {
 511         jobs {
 512             name(job.name)
 513         }
 514     }
 515
 516     // Add to OS view.
 517     Views.OSViews[os].with {
 518         jobs {
 519             name(job.name)
 520         }
 521     }
 522 }
 523
 524 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 525     addToPeriodicView(job)
 526     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 527 }
 528
 529 def static addGithubPushTriggerHelper(def job) {
 530     addToMergeView(job)
 531     Utilities.addGithubPushTrigger(job)
 532 }
 533
 534
 535 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 536     assert os instanceof String
 537     assert architecture instanceof String
 538
 539     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 540     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 541
 542     if (!(architecture in armArches)) {
 543         assert options == null
 544         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 545
 546         return
 547     }
 548
 549     // This is an arm(64) job.
 550     //
 551     // There are several options.
 552     //
 553     // Windows_NT
 554     //
 555     // Arm32 (Build) -> latest-arm64
 556     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 557     // Arm32 (Test)  -> arm64-windows_nt
 558     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 559     //
 560     // Arm64 (Build) -> latest-arm64
 561     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 562     // Arm64 (Test)  -> arm64-windows_nt
 563     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 564     //
 565     // Ubuntu
 566     //
 567     // Arm32 emulator (Build, Test) -> arm-cross-latest
 568     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 569     //
 570     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 571     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 572     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 573     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 574     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 575     //       |-> os == "Ubuntu" && (architecture == "arm")
 576     //
 577     // Arm64 (Build) -> arm64-cross-latest
 578     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 579     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 580     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 581     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 582     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 583
 584     // This has to be a arm arch
 585     assert architecture in armArches
 586     if (os == "Windows_NT") {
 587         // arm32/arm64 Windows jobs share the same machines for now
 588         def isBuild = options['use_arm64_build_machine'] == true
 589
 590         if (isBuild == true) {
 591             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 592         } else {
 593             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 594         }
 595     } else {
 596         assert os != 'Windows_NT'
 597         assert os in supportedArmLinuxOs
 598
 599         if (architecture == 'arm64') {
 600             if ((options != null) && (options['is_build_only'] == true)) {
 601                 // Arm64 Linux build machine
 602                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 603             } else {
 604                 // Arm64 Linux test machines
 605                 if ((options != null) && (options['large_pages'] == true)) {
 606                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 607                 } else {
 608                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 609                 }
 610             }
 611         }
 612         else if (architecture == 'armem') {
 613             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 614             // using Docker.
 615             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 616         }
 617         else {
 618             // arm Ubuntu on hardware.
 619             assert (architecture == 'arm') && (os == 'Ubuntu')
 620             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 621             def isBuild = (options != null) && (options['is_build_job'] == true)
 622             if (isFlow || isBuild) {
 623                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 624                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 625                 // arm hardware.
 626                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 627             } else {
 628                 // arm Ubuntu test machine
 629                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 630                 // Utilities.setMachineAffinity. Just add the machine affinity
 631                 // manually. We specify the Helix queue name here.
 632                 job.with {
 633                     label('ubuntu.1404.arm32.open')
 634                 }
 635             }
 636         }
 637     }
 638 }
 639
 640 // setJobMachineAffinity: compute the machine affinity options for a job,
 641 // then set the job with those affinity options.
 642 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 643 {
 644     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 645            (!isBuildJob && isTestJob  && !isFlowJob) ||
 646            (!isBuildJob && !isTestJob && isFlowJob)
 647
 648     def affinityOptions = null
 649     def affinityArchitecture = architecture
 650
 651     if (os == "Windows_NT") {
 652         if (architecture in Constants.armWindowsCrossArchitectureList) {
 653             if (isBuildJob) {
 654                 affinityOptions = [ "use_arm64_build_machine" : true ]
 655             } else if (isTestJob) {
 656                 affinityOptions = [ "use_arm64_build_machine" : false ]
 657             } else if (isFlowJob) {
 658                 // For the flow jobs set the machine affinity as x64
 659                 affinityArchitecture = 'x64'
 660             }
 661         }
 662     }
 663     else {
 664         if (architecture == 'arm64') {
 665             if (isBuildJob) {
 666                 affinityOptions = ['is_build_only': true]
 667             } else if (isTestJob) {
 668                 affinityOptions = [ "large_pages" : false ]
 669             }
 670         }
 671         else if (architecture == 'arm') {
 672             if (isBuildJob) {
 673                 affinityOptions = ['is_build_job': true]
 674             } else if (isFlowJob) {
 675                 affinityOptions = ['is_flow_job': true]
 676             }
 677         }
 678     }
 679
 680     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 681 }
 682
 683 def static isGCStressRelatedTesting(def scenario) {
 684     // The 'r2r_gcstress15' scenario is a basic scenario.
 685     // Detect it and make it a GCStress related.
 686     if (scenario == 'r2r_gcstress15')
 687     {
 688         return true;
 689     }
 690
 691     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 692     def scenarioName = scenario.toLowerCase()
 693     def isGCStressTesting = false
 694     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 695         if (k in gcStressTestEnvVars) {
 696             isGCStressTesting = true;
 697         }
 698     }
 699     return isGCStressTesting
 700 }
 701
 702 def static isCoreFxScenario(def scenario) {
 703     def corefx_prefix = 'corefx_'
 704     if (scenario.length() < corefx_prefix.length()) {
 705         return false
 706     }
 707     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 708 }
 709
 710 def static isR2RBaselineScenario(def scenario) {
 711     return (scenario == 'r2r')
 712 }
 713
 714 def static isR2RStressScenario(def scenario) {
 715     return Constants.r2rStressScenarios.containsKey(scenario)
 716 }
 717
 718 def static isR2RScenario(def scenario) {
 719     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 720 }
 721
 722 def static isJitStressScenario(def scenario) {
 723     return Constants.jitStressModeScenarios.containsKey(scenario)
 724 }
 725
 726 def static isLongGc(def scenario) {
 727     return (scenario == 'longgc' || scenario == 'gcsimulator')
 728 }
 729
 730 def static isJitDiff(def scenario) {
 731     return (scenario == 'jitdiff')
 732 }
 733
 734 def static isGcReliabilityFramework(def scenario) {
 735     return (scenario == 'gc_reliability_framework')
 736 }
 737
 738 def static isArmWindowsScenario(def scenario) {
 739     return Constants.validArmWindowsScenarios.containsKey(scenario)
 740 }
 741
 742 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 743     if (isBuildOnly == true) {
 744         os = 'Windows_NT_BuildOnly'
 745     }
 746
 747     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 748
 749     if (validOsPrTriggerArchConfigs == null) {
 750         return false
 751     }
 752
 753     if (validOsPrTriggerArchConfigs[architecture] != null) {
 754         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 755
 756         if (!(configuration in validOsPrTriggerConfigs)) {
 757             return false
 758         }
 759     } else {
 760         return false
 761     }
 762
 763     return true
 764 }
 765
 766 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 767     // 2 hours (120 minutes) is the default timeout
 768     def timeout = 120
 769     def innerLoop = (scenario == "innerloop")
 770
 771     if (!innerLoop) {
 772         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 773         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 774         timeout = 240
 775     }
 776
 777     if (!isBuildOnly) {
 778         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 779         if (isGCStressRelatedTesting(scenario)) {
 780             timeout = 4320
 781         }
 782         else if (isCoreFxScenario(scenario)) {
 783             timeout = 360
 784         }
 785         else if (isJitStressScenario(scenario)) {
 786             timeout = 300
 787         }
 788         else if (isR2RBaselineScenario(scenario)) {
 789             timeout = 240
 790         }
 791         else if (isLongGc(scenario)) {
 792             timeout = 1440
 793         }
 794         else if (isJitDiff(scenario)) {
 795             timeout = 240
 796         }
 797         else if (isGcReliabilityFramework(scenario)) {
 798             timeout = 1440
 799         }
 800         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 801             timeout = 240
 802         }
 803
 804         if (architecture == 'arm') {
 805             // ARM32 machines are particularly slow.
 806             timeout += 120
 807         }
 808     }
 809
 810     if (configuration == 'Debug') {
 811         // Debug runs can be very slow. Add an hour.
 812         timeout += 60
 813     }
 814
 815     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 816         // AltJit runs compile all methods twice.
 817         timeout *= 2
 818     }
 819
 820     // If we've changed the timeout from the default, set it in the job.
 821
 822     if (timeout != 120) {
 823         Utilities.setJobTimeout(newJob, timeout)
 824     }
 825 }
 826
 827 def static getJobFolder(def scenario) {
 828     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 829         return 'jitstress'
 830     }
 831     if (scenario == 'illink') {
 832         return 'illink'
 833     }
 834     return ''
 835 }
 836
 837 def static getStressModeDisplayName(def scenario) {
 838     def displayStr = ''
 839     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 840         def prefixLength = 'COMPlus_'.length()
 841         if (k.length() >= prefixLength) {
 842             def modeName = k.substring(prefixLength, k.length())
 843             displayStr += ' ' + modeName + '=' + v
 844         }
 845     }
 846
 847     if (isCoreFxScenario(scenario)) {
 848         displayStr = ('CoreFx ' + displayStr).trim()
 849     }
 850
 851     return displayStr
 852 }
 853
 854 def static getR2RDisplayName(def scenario) {
 855     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 856     def displayStr = scenario
 857     def prefixLength = 'r2r_'.length()
 858     if (displayStr.length() >= prefixLength) {
 859         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 860     } else if (scenario == 'r2r') {
 861         displayStr = "R2R"
 862     }
 863     return displayStr
 864 }
 865
 866 //
 867 // Functions to create an environment script.
 868 //      envScriptCreate -- initialize the script (call first)
 869 //      envScriptFinalize -- finalize the script (call last)
 870 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 871 //      envScriptAppendExistingScript -- append an existing script to the generated script
 872 //
 873 // Each script returns a string of commands. Concatenate all the strings together before
 874 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 875 //
 876
 877 // Initialize the environment setting script.
 878 def static envScriptCreate(def os, def stepScriptLocation) {
 879     def stepScript = ''
 880     if (os == 'Windows_NT') {
 881         stepScript += "echo Creating TestEnv script\r\n"
 882         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 883
 884         // Create at least an empty script.
 885         stepScript += "echo. > ${stepScriptLocation}\r\n"
 886     }
 887     else {
 888         stepScript += "echo Creating environment setting script\n"
 889         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 890     }
 891
 892     return stepScript
 893 }
 894
 895 // Generates the string for setting stress mode variables.
 896 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 897     def stepScript = ''
 898     if (os == 'Windows_NT') {
 899         stressModeVars.each{ k, v ->
 900             // Write out what we are writing to the script file
 901             stepScript += "echo Setting ${k}=${v}\r\n"
 902             // Write out the set itself to the script file`
 903             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 904         }
 905     }
 906     else {
 907         stressModeVars.each{ k, v ->
 908             // Write out what we are writing to the script file
 909             stepScript += "echo Setting ${k}=${v}\n"
 910             // Write out the set itself to the script file`
 911             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 912         }
 913     }
 914
 915     return stepScript
 916 }
 917
 918 // Append an existing script to an environment script.
 919 // Returns string of commands to do this.
 920 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 921     assert (os == 'Windows_NT')
 922     def stepScript = ''
 923
 924     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 925     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 926
 927     return stepScript
 928 }
 929
 930 // Finalize an environment setting script.
 931 // Returns string of commands to do this.
 932 def static envScriptFinalize(def os, def stepScriptLocation) {
 933     def stepScript = ''
 934
 935     if (os == 'Windows_NT') {
 936         // Display the resulting script. This is useful when looking at the output log file.
 937         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 938         stepScript += "type ${stepScriptLocation}\r\n"
 939     }
 940     else {
 941         stepScript += "chmod +x ${stepScriptLocation}\n"
 942     }
 943
 944     return stepScript
 945 }
 946
 947 def static isNeedDocker(def architecture, def os, def isBuild) {
 948     if (isBuild) {
 949         if (architecture == 'x86' && os == 'Ubuntu') {
 950             return true
 951         }
 952         else if (architecture == 'armem') {
 953             return true
 954         }
 955         else if (architecture == 'arm') {
 956             if (os == 'Ubuntu') {
 957                 return true
 958             }
 959         }
 960     }
 961     else {
 962         if (architecture == 'x86' && os == 'Ubuntu') {
 963             return true
 964         }
 965     }
 966     return false
 967 }
 968
 969 def static getDockerImageName(def architecture, def os, def isBuild) {
 970     // We must change some docker private images to official later
 971     if (isBuild) {
 972         if (architecture == 'x86' && os == 'Ubuntu') {
 973             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 974         }
 975         else if (architecture == 'armem') {
 976             if (os == 'Ubuntu') {
 977                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 978             }
 979             else if (os == 'Ubuntu16.04') {
 980                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 981             }
 982             else if (os == 'Tizen') {
 983                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 984             }
 985         }
 986         else if (architecture == 'arm') {
 987             if (os == 'Ubuntu') {
 988                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180323032140"
 989             }
 990         }
 991     }
 992     else {
 993         if (architecture == 'x86' && os == 'Ubuntu') {
 994             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 995         }
 996     }
 997     println("Unknown architecture to use docker: ${architecture} ${os}");
 998     assert false
 999 }
1000
1001
1002 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
1003 def static jobRequiresLimitedHardware(def architecture, def os) {
1004     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
1005         // These test jobs require ARM64 hardware
1006         return true
1007     }
1008     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1009         // These test jobs require Linux/arm32 hardware
1010         return true
1011     }
1012     else {
1013         return false
1014     }
1015 }
1016
1017 // Calculates the name of the build job based on some typical parameters.
1018 //
1019 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1020     // If the architecture is x64, do not add that info into the build name.
1021     // Need to change around some systems and other builds to pick up the right builds
1022     // to do that.
1023
1024     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1025     if (isBuildOnly) {
1026         suffix += '_bld'
1027     }
1028     def baseName = ''
1029     switch (architecture) {
1030         case 'x64':
1031             if (scenario == 'normal') {
1032                 // For now we leave x64 off of the name for compatibility with other jobs
1033                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1034             }
1035             else if (scenario == 'formatting') {
1036                 // we don't care about the configuration for the formatting job. It runs all configs
1037                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1038             }
1039             else {
1040                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1041             }
1042             break
1043         case 'arm64':
1044             if (os.toLowerCase() == "windows_nt") {
1045                 // These are cross builds
1046                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1047             }
1048             else {
1049                 // Defaults to a small page size set of machines.
1050                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1051             }
1052             break
1053         case 'armem':
1054             // These are cross builds
1055             if (os == 'Tizen') {
1056                 // ABI: softfp
1057                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1058             }
1059             else {
1060                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1061             }
1062             break
1063         case 'armlb':
1064         case 'arm':
1065             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1066             break
1067         case 'x86':
1068         case 'x86_arm_altjit':
1069         case 'x64_arm64_altjit':
1070             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1071             break
1072         default:
1073             println("Unknown architecture: ${architecture}");
1074             assert false
1075             break
1076     }
1077
1078     return baseName + suffix
1079 }
1080
1081 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1082
1083     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1084     if (os == 'Windows_NT') {
1085         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1086             if (!(branch in Constants.WindowsArm64Branches)) {
1087                 return
1088             }
1089         }
1090     }
1091
1092     // Check scenario.
1093     switch (scenario) {
1094         case 'innerloop':
1095             break
1096         case 'normal':
1097             switch (architecture) {
1098                 case 'x64':
1099                 case 'x86':
1100                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1101                         addPeriodicTriggerHelper(job, '@daily')
1102                     }
1103                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     break
1107                 case 'arm':
1108                     if (os == 'Windows_NT') {
1109                         addGithubPushTriggerHelper(job)
1110                     }
1111                     else {
1112                         // Currently no push triggers, with limited arm Linux hardware.
1113                         // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1114                         assert os == 'Ubuntu'
1115                         if (isFlowJob) {
1116                             addPeriodicTriggerHelper(job, '@daily')
1117                         }
1118                     }
1119                     break
1120                 case 'armem':
1121                 case 'armlb':
1122                 case 'x86_arm_altjit':
1123                 case 'x64_arm64_altjit':
1124                     addGithubPushTriggerHelper(job)
1125                     break
1126                 case 'arm64':
1127                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1128                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1129                     break
1130                 default:
1131                     println("Unknown architecture: ${architecture}");
1132                     assert false
1133                     break
1134             }
1135             break
1136         case 'r2r':
1137             assert !(os in bidailyCrossList)
1138             // r2r gets a push trigger for checked/release
1139             if (configuration == 'Checked' || configuration == 'Release') {
1140                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1141                 if (architecture == 'x64' && os != 'OSX10.12') {
1142                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1143                     if (isFlowJob || os == 'Windows_NT') {
1144                         addGithubPushTriggerHelper(job)
1145                     }
1146                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1147                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1148                     if (isFlowJob) {
1149                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1150                     }
1151                 }
1152                 // For x86, only add per-commit jobs for Windows
1153                 else if (architecture == 'x86') {
1154                     if (os == 'Windows_NT') {
1155                         addGithubPushTriggerHelper(job)
1156                     }
1157                 }
1158                 // arm64 r2r jobs should only run daily.
1159                 else if (architecture == 'arm64') {
1160                     if (os == 'Windows_NT') {
1161                         addPeriodicTriggerHelper(job, '@daily')
1162                     }
1163                 }
1164             }
1165             break
1166         case 'r2r_jitstress1':
1167         case 'r2r_jitstress2':
1168         case 'r2r_jitstressregs1':
1169         case 'r2r_jitstressregs2':
1170         case 'r2r_jitstressregs3':
1171         case 'r2r_jitstressregs4':
1172         case 'r2r_jitstressregs8':
1173         case 'r2r_jitstressregs0x10':
1174         case 'r2r_jitstressregs0x80':
1175         case 'r2r_jitstressregs0x1000':
1176         case 'r2r_jitminopts':
1177         case 'r2r_jitforcerelocs':
1178         case 'r2r_gcstress15':
1179             assert !(os in bidailyCrossList)
1180
1181             // GCStress=C is currently not supported on OS X
1182             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1183                 break
1184             }
1185
1186             // GC Stress 15 r2r gets a push trigger for checked/release
1187             if (configuration == 'Checked' || configuration == 'Release') {
1188                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1189                 if (architecture == 'x64') {
1190                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1191                     if (isFlowJob || os == 'Windows_NT') {
1192                         // Add a weekly periodic trigger
1193                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1194                     }
1195                 }
1196                 // For x86, only add per-commit jobs for Windows
1197                 else if (architecture == 'x86') {
1198                     if (os == 'Windows_NT') {
1199                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1200                     }
1201                 }
1202             }
1203             break
1204         case 'longgc':
1205             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1206             assert configuration == 'Release'
1207             assert architecture == 'x64'
1208             addPeriodicTriggerHelper(job, '@daily')
1209             // TODO: Add once external email sending is available again
1210             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1211             break
1212         case 'gcsimulator':
1213             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1214             assert configuration == 'Release'
1215             assert architecture == 'x64'
1216             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1217             // TODO: Add once external email sending is available again
1218             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1219             break
1220         case 'standalone_gc':
1221             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1222             assert (configuration == 'Release' || configuration == 'Checked')
1223             // TODO: Add once external email sending is available again
1224             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1225             addPeriodicTriggerHelper(job, '@daily')
1226             break
1227         case 'gc_reliability_framework':
1228             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1229             assert (configuration == 'Release' || configuration == 'Checked')
1230             // Only triggered by phrase.
1231             break
1232         case 'ilrt':
1233             assert !(os in bidailyCrossList)
1234             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1235             if (architecture == 'x64' && configuration == 'Release') {
1236                 // We don't expect to see a job generated except in these scenarios
1237                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1238                 if (isFlowJob || os == 'Windows_NT') {
1239                     addPeriodicTriggerHelper(job, '@daily')
1240                 }
1241             }
1242             break
1243         case 'jitdiff':
1244             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1245             assert configuration == 'Checked'
1246             assert (architecture == 'x64' || architecture == 'x86')
1247             addGithubPushTriggerHelper(job)
1248             break
1249         case 'formatting':
1250             assert (os == 'Windows_NT' || os == "Ubuntu")
1251             assert architecture == 'x64'
1252             addGithubPushTriggerHelper(job)
1253             break
1254         case 'jitstressregs1':
1255         case 'jitstressregs2':
1256         case 'jitstressregs3':
1257         case 'jitstressregs4':
1258         case 'jitstressregs8':
1259         case 'jitstressregs0x10':
1260         case 'jitstressregs0x80':
1261         case 'jitstressregs0x1000':
1262         case 'minopts':
1263         case 'forcerelocs':
1264         case 'jitstress1':
1265         case 'jitstress2':
1266         case 'jitstress2_jitstressregs1':
1267         case 'jitstress2_jitstressregs2':
1268         case 'jitstress2_jitstressregs3':
1269         case 'jitstress2_jitstressregs4':
1270         case 'jitstress2_jitstressregs8':
1271         case 'jitstress2_jitstressregs0x10':
1272         case 'jitstress2_jitstressregs0x80':
1273         case 'jitstress2_jitstressregs0x1000':
1274         case 'tailcallstress':
1275         case 'jitsse2only':
1276         case 'jitnosimd':
1277         case 'jitnox86hwintrinsic':
1278         case 'jitincompletehwintrinsic':
1279         case 'jitx86hwintrinsicnoavx':
1280         case 'jitx86hwintrinsicnoavx2':
1281         case 'jitx86hwintrinsicnosimd':
1282         case 'corefx_baseline':
1283         case 'corefx_minopts':
1284         case 'corefx_jitstress1':
1285         case 'corefx_jitstress2':
1286         case 'corefx_jitstressregs1':
1287         case 'corefx_jitstressregs2':
1288         case 'corefx_jitstressregs3':
1289         case 'corefx_jitstressregs4':
1290         case 'corefx_jitstressregs8':
1291         case 'corefx_jitstressregs0x10':
1292         case 'corefx_jitstressregs0x80':
1293         case 'corefx_jitstressregs0x1000':
1294         case 'zapdisable':
1295             if (os == 'CentOS7.1') {
1296                 break
1297             }
1298             if (os in bidailyCrossList) {
1299                 break
1300             }
1301             assert (os == 'Windows_NT') || (os in Constants.crossList)
1302             if (jobRequiresLimitedHardware(architecture, os)) {
1303                 addPeriodicTriggerHelper(job, '@weekly')
1304             }
1305             else {
1306                 addPeriodicTriggerHelper(job, '@daily')
1307             }
1308             break
1309         case 'heapverify1':
1310         case 'gcstress0x3':
1311             if (os == 'CentOS7.1') {
1312                 break
1313             }
1314             if (os in bidailyCrossList) {
1315                 break
1316             }
1317             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1318                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1319                 break
1320             }
1321             assert (os == 'Windows_NT') || (os in Constants.crossList)
1322             addPeriodicTriggerHelper(job, '@weekly')
1323             break
1324         case 'gcstress0xc':
1325         case 'gcstress0xc_zapdisable':
1326         case 'gcstress0xc_zapdisable_jitstress2':
1327         case 'gcstress0xc_zapdisable_heapverify1':
1328         case 'gcstress0xc_jitstress1':
1329         case 'gcstress0xc_jitstress2':
1330         case 'gcstress0xc_minopts_heapverify1':
1331             if (os == 'CentOS7.1') {
1332                 break
1333             }
1334             if (os == 'OSX10.12') {
1335                 // GCStress=C is currently not supported on OS X
1336                 break
1337             }
1338             if (os in bidailyCrossList) {
1339                 break
1340             }
1341             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1342                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1343                 break
1344             }
1345             assert (os == 'Windows_NT') || (os in Constants.crossList)
1346             addPeriodicTriggerHelper(job, '@weekly')
1347             break
1348
1349         case 'illink':
1350             // Testing on other operating systems TBD
1351             assert (os == 'Windows_NT' || os == 'Ubuntu')
1352             if (architecture == 'x64' || architecture == 'x86') {
1353                 if (configuration == 'Checked') {
1354                     addPeriodicTriggerHelper(job, '@daily')
1355                 }
1356             }
1357             break
1358
1359         case 'tieredcompilation':
1360         case 'corefx_tieredcompilation':
1361             // No periodic jobs just yet, still testing
1362             break
1363
1364         default:
1365             println("Unknown scenario: ${scenario}");
1366             assert false
1367             break
1368     }
1369     return
1370 }
1371
1372 // **************************
1373 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1374 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1375 // builds will do a couple extra steps.
1376 // **************************
1377
1378 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1379 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1380 // it's just used for internal builds
1381 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1382 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1383     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1384
1385     if (isWindowsBuildOnlyJob) {
1386         return
1387     }
1388
1389     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1390     // Non pull request builds.
1391     if (!isPR) {
1392         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1393         return
1394     }
1395
1396      def arm64Users = [
1397         'AndyAyersMS',
1398         'briansull',
1399         'BruceForstall',
1400         'CarolEidt',
1401         'cmckinsey',
1402         'erozenfeld',
1403         'janvorli',
1404         'jashook',
1405         'JosephTremoulet',
1406         'pgodeq',
1407         'pgavlin',
1408         'rartemev',
1409         'russellhadley',
1410         'RussKeldorph',
1411         'sandreenko',
1412         'sdmaclea',
1413         'swaroop-sridhar',
1414         'jkotas',
1415         'markwilkie',
1416         'weshaggard'
1417     ]
1418
1419     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1420     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1421     def osGroup = getOSGroup(os)
1422     switch (architecture) {
1423         case 'x64': // editor brace matching: {
1424             if (scenario == 'formatting') {
1425                 assert configuration == 'Checked'
1426                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1427                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1428                 }
1429
1430                 break
1431             }
1432
1433             switch (os) {
1434                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1435                 case 'Debian8.4':
1436                 case 'RHEL7.2':
1437                     if (scenario == 'innerloop') {
1438                         assert !isFlowJob
1439                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1440                     }
1441                     else if (scenario == 'normal') {
1442                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1443                     }
1444                     break
1445
1446                 case 'Ubuntu16.04':
1447                     assert !isFlowJob
1448                     assert scenario != 'innerloop'
1449                     // Distinguish with the other architectures (arm and x86)
1450                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1451                     break
1452
1453                 case 'Fedora24':
1454                 case 'Ubuntu16.10':
1455                     assert !isFlowJob
1456                     assert scenario != 'innerloop'
1457                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1458                     break
1459
1460                 case 'Ubuntu':
1461                     if (scenario == 'illink') {
1462                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1463                         break
1464                     }
1465                     // fall through
1466
1467                 case 'OSX10.12':
1468                     // Triggers on the non-flow jobs aren't necessary here
1469                     // Corefx testing uses non-flow jobs.
1470                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1471                         break
1472                     }
1473                     switch (scenario) {
1474                         case 'innerloop':
1475                             // PR Triggered jobs. These jobs will run pri0 tests.
1476                             if (configuration == 'Checked') {
1477                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1478                             }
1479                             break
1480
1481                         case 'normal':
1482                             // OSX uses checked for default PR tests
1483                             if (configuration == 'Checked') {
1484                                 // Default trigger
1485                                 assert !job.name.contains("centos")
1486                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1487                             }
1488                             break
1489
1490                         case 'jitdiff':
1491                             if (configuration == 'Checked') {
1492                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1493                             }
1494                             break
1495
1496                         case 'ilrt':
1497                             if (configuration == 'Release') {
1498                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1499                             }
1500                             break
1501
1502                         case 'longgc':
1503                             if (configuration == 'Release') {
1504                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1505                             }
1506                             break
1507
1508                         case 'gcsimulator':
1509                             if (configuration == 'Release') {
1510                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1511                             }
1512                             break
1513
1514                         case 'standalone_gc':
1515                             if (configuration == 'Release' || configuration == 'Checked') {
1516                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1517                             }
1518                             break
1519
1520                         case 'gc_reliability_framework':
1521                             if (configuration == 'Release' || configuration == 'Checked') {
1522                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1523                             }
1524                             break
1525
1526                         default:
1527                             if (isJitStressScenario(scenario)) {
1528                                 def displayStr = getStressModeDisplayName(scenario)
1529                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1530                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1531                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1532                             }
1533                             else if (isR2RScenario(scenario)) {
1534                                 if (configuration == 'Release' || configuration == 'Checked') {
1535                                     def displayStr = getR2RDisplayName(scenario)
1536                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1537                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1538                                 }
1539                             }
1540                             else {
1541                                 println("Unknown scenario: ${scenario}");
1542                                 assert false
1543                             }
1544                             break
1545
1546                     }
1547                     break
1548
1549                 case 'CentOS7.1':
1550                     switch (scenario) {
1551                         case 'innerloop':
1552                             // CentOS uses checked for default PR tests while debug is build only
1553                             if (configuration == 'Debug') {
1554                                 // Default trigger
1555                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1556                             }
1557
1558                             // Make sure this is a flow job to get build and test.
1559                             if (configuration == 'Checked' && isFlowJob) {
1560                                 assert job.name.contains("flow")
1561                                 // Default trigger
1562                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1563                             }
1564                             break
1565
1566                         case 'normal':
1567                             // Make sure this is a flow job to get build and test.
1568                             if (configuration == 'Checked' && isFlowJob) {
1569                                 assert job.name.contains("flow")
1570                                 // Default trigger
1571                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1572                             }
1573                             break
1574
1575                         default:
1576                             if (isR2RScenario(scenario)) {
1577                                 if (configuration == 'Release' || configuration == 'Checked') {
1578                                     def displayStr = getR2RDisplayName(scenario)
1579                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1580                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1581                                 }
1582                             }
1583                             break
1584
1585                     }
1586                     break
1587
1588                 case 'Windows_NT':
1589                     switch (scenario) {
1590                         case 'innerloop':
1591                             // Default trigger
1592                             if (configuration == 'Checked' || configuration == 'Release') {
1593                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1594                             }
1595                             break
1596
1597                         case 'normal':
1598                             if (configuration == 'Checked') {
1599                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1600                             }
1601                             break
1602
1603                         case 'jitdiff':
1604                             if (configuration == 'Checked') {
1605                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1606                             }
1607                             break
1608
1609                         case 'ilrt':
1610                             if (configuration == 'Release') {
1611                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1612                             }
1613                             break
1614
1615                         case 'longgc':
1616                             if (configuration == 'Release') {
1617                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1618                             }
1619                             break
1620
1621                         case 'gcsimulator':
1622                             if (configuration == 'Release') {
1623                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1624                             }
1625                             break
1626
1627                         case 'standalone_gc':
1628                             if (configuration == 'Release' || configuration == 'Checked') {
1629                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1630                             }
1631                             break
1632
1633                         case 'gc_reliability_framework':
1634                             if (configuration == 'Release' || configuration == 'Checked') {
1635                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1636                             }
1637                             break
1638
1639                         case 'illink':
1640                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1641                             break
1642
1643                         default:
1644                             if (isJitStressScenario(scenario)) {
1645                                 def displayStr = getStressModeDisplayName(scenario)
1646                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1647                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1648                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1649                             }
1650                             else if (isR2RScenario(scenario)) {
1651                                 if (configuration == 'Release' || configuration == 'Checked') {
1652                                     def displayStr = getR2RDisplayName(scenario)
1653                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1654                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1655                                 }
1656                             }
1657                             else {
1658                                 println("Unknown scenario: ${scenario}");
1659                                 assert false
1660                             }
1661                             break
1662
1663                     }
1664                     break
1665
1666                 default:
1667                     println("Unknown os: ${os}");
1668                     assert false
1669                     break
1670
1671             }
1672
1673             break
1674         // editor brace matching: }
1675
1676         case 'armem': // editor brace matching: {
1677             job.with {
1678                 publishers {
1679                     azureVMAgentPostBuildAction {
1680                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1681                     }
1682                 }
1683             }
1684
1685             switch (os) {
1686                 case 'Ubuntu':
1687                 case 'Ubuntu16.04':
1688                     assert scenario != 'innerloop'
1689                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1690                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1691                     break
1692
1693                 case 'Tizen':
1694                     architecture = 'armel'
1695
1696                     if (scenario == 'innerloop') {
1697                         if (configuration == 'Checked') {
1698                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1699                         }
1700                     }
1701                     else {
1702                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1703                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1704                     }
1705                     break
1706             }
1707
1708             break
1709         // editor brace matching: }
1710
1711         case 'armlb':
1712         case 'arm': // editor brace matching: {
1713
1714             // Triggers on the non-flow jobs aren't necessary
1715             if (!isFlowJob) {
1716                 break
1717             }
1718
1719             // Set up a private trigger
1720             def contextString = "${os} ${architecture} Cross ${configuration}"
1721             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1722             if (scenario == 'innerloop') {
1723                 contextString += " Innerloop"
1724                 triggerString += "\\W+Innerloop"
1725             }
1726             else {
1727                 contextString += " ${scenario}"
1728                 triggerString += "\\W+${scenario}"
1729             }
1730
1731             if (configuration == 'Debug') {
1732                 contextString += " Build"
1733                 triggerString += "\\W+Build"
1734             } else {
1735                 contextString += " Build and Test"
1736                 triggerString += "\\W+Build and Test"
1737             }
1738
1739             triggerString += ".*"
1740
1741             switch (os) {
1742                 case 'Ubuntu':
1743                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1744                         break
1745                     }
1746
1747                     if (scenario == 'innerloop') {
1748                         if (configuration == 'Checked') {
1749                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1750                         }
1751                     }
1752                     else {
1753                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1754                     }
1755                     break
1756
1757                 case 'Windows_NT':
1758                     if (architecture == "armlb") {
1759                         // Disable armlb windows jobs
1760                         break
1761                     }
1762                     switch (scenario) {
1763                         case 'innerloop':
1764                             // Only Checked is an innerloop trigger.
1765                             if (configuration == 'Checked')
1766                             {
1767                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1768                             }
1769                             break
1770                         case 'normal':
1771                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1772                             break
1773                         default:
1774                             // Stress jobs will use this code path.
1775                             if (isArmWindowsScenario(scenario)) {
1776                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1777                             }
1778                             break
1779                     }
1780                     break
1781                 default:
1782                     println("NYI os: ${os}");
1783                     assert false
1784                     break
1785             }
1786             break
1787         // editor brace matching: }
1788         case 'arm64': // editor brace matching: {
1789             // Set up a private trigger
1790             def contextString = "${os} ${architecture} Cross ${configuration}"
1791             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1792
1793             if (scenario == 'innerloop') {
1794                 contextString += " Innerloop"
1795                 triggerString += "\\W+Innerloop"
1796             }
1797             else {
1798                 contextString += " ${scenario}"
1799                 triggerString += "\\W+${scenario}"
1800             }
1801
1802             if (configuration == 'Debug') {
1803                 contextString += " Build"
1804                 triggerString += "\\W+Build"
1805             } else {
1806                 contextString += " Build and Test"
1807                 triggerString += "\\W+Build and Test"
1808             }
1809
1810             triggerString += ".*"
1811
1812             switch (os) {
1813                 case 'Ubuntu':
1814                 case 'Ubuntu16.04':
1815                     switch (scenario) {
1816                         case 'innerloop':
1817                             if (configuration == 'Debug' && !isFlowJob) {
1818                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1819                             }
1820
1821                             break
1822                         case 'normal':
1823                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1824                             break
1825                         default:
1826                             if (isR2RScenario(scenario)) {
1827                                 if (configuration == 'Checked' || configuration == 'Release') {
1828                                     def displayStr = getR2RDisplayName(scenario)
1829                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1830                                 }
1831                             }
1832                             break
1833                     }
1834                     break
1835
1836                 case 'Windows_NT':
1837                     // Triggers on the non-flow jobs aren't necessary here
1838                     if (!isFlowJob) {
1839                         break
1840                     }
1841
1842                     assert isArmWindowsScenario(scenario)
1843                     switch (scenario) {
1844                         case 'innerloop':
1845                             if (configuration == 'Checked') {
1846                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1847                             }
1848
1849                             break
1850                         case 'normal':
1851                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1852                             break
1853                         default:
1854                             // Stress jobs will use this code path.
1855                             if (isArmWindowsScenario(scenario)) {
1856                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1857                             }
1858                             break
1859                     }
1860                     break
1861                 default:
1862                     println("NYI os: ${os}");
1863                     assert false
1864                     break
1865             }
1866             break
1867
1868         // editor brace matching: }
1869         case 'x86': // editor brace matching: {
1870             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1871             if (os == 'Ubuntu') {
1872                 // Triggers on the non-flow jobs aren't necessary here
1873                 if (!isFlowJob) {
1874                     break
1875                 }
1876
1877                 // on-demand only for ubuntu x86
1878                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1879                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1880                 break
1881
1882             }
1883             switch (scenario) {
1884                 case 'innerloop':
1885                     if (configuration == 'Checked' || configuration == 'Release') {
1886                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1887                     }
1888                     break
1889
1890                 case 'normal':
1891                     if (configuration == 'Checked') {
1892                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1893                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1894                     }
1895                     break
1896
1897                 case 'ilrt':
1898                     if (configuration == 'Release') {
1899                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1900                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1901                     }
1902                     break
1903
1904                 case 'longgc':
1905                     if (configuration == 'Release') {
1906                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1907                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1908                     }
1909                     break
1910
1911                 case 'gcsimulator':
1912                     if (configuration == 'Release') {
1913                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1914                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1915                     }
1916                     break
1917
1918                 case 'standalone_gc':
1919                     if (configuration == 'Release' || configuration == 'Checked') {
1920                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1921                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1922                     }
1923                     break
1924
1925                 case 'illink':
1926                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1927                     break
1928
1929                 default:
1930                     if (isJitStressScenario(scenario)) {
1931                         def displayStr = getStressModeDisplayName(scenario)
1932                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1933                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1934                     }
1935                     else if (isR2RScenario(scenario)) {
1936                         if (configuration == 'Release' || configuration == 'Checked') {
1937                             def displayStr = getR2RDisplayName(scenario)
1938                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1939                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1940                         }
1941                     }
1942                     else {
1943                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1944                         assert false
1945                     }
1946                     break
1947
1948             }
1949             break
1950
1951          // editor brace matching: }
1952         case 'x64_arm64_altjit':
1953         case 'x86_arm_altjit': // editor brace matching: {
1954             assert (os == 'Windows_NT')
1955             switch (scenario) {
1956                 case 'normal':
1957                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1958                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1959                     break
1960                 default:
1961                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1962                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1963                     break
1964             }
1965             break
1966
1967         // editor brace matching: }
1968         default:
1969             println("Unknown architecture: ${architecture}");
1970             assert false
1971             break
1972     }
1973 }
1974
1975 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1976     def buildCommands = []
1977     def osGroup = getOSGroup(os)
1978     def lowerConfiguration = configuration.toLowerCase()
1979
1980     def priority = '1'
1981     if (scenario == 'innerloop') {
1982         priority = '0'
1983     }
1984
1985     def doCoreFxTesting = isCoreFxScenario(scenario)
1986
1987     // Calculate the build steps, archival, and xunit results
1988     switch (os) {
1989         case 'Windows_NT': // editor brace matching: {
1990             switch (architecture) {
1991                 case 'x64':
1992                 case 'x86':
1993                 case 'x86_arm_altjit':
1994                 case 'x64_arm64_altjit':
1995                     def arch = architecture
1996                     def buildOpts = ''
1997                     if (architecture == 'x86_arm_altjit') {
1998                         arch = 'x86'
1999                     }
2000                     else if (architecture == 'x64_arm64_altjit') {
2001                         arch = 'x64'
2002                     }
2003
2004                     if (scenario == 'formatting') {
2005                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2006                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2007                         break
2008                     }
2009
2010                     if (scenario == 'illink') {
2011                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2012                     }
2013
2014                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2015                     if ((lowerConfiguration == 'release') &&
2016                         (scenario in Constants.basicScenarios) &&
2017                         (architecture != 'x86_arm_altjit') &&
2018                         (architecture != 'x64_arm64_altjit')) {
2019
2020                         buildOpts += ' -enforcepgo'
2021                     }
2022
2023                     if (doCoreFxTesting) {
2024                         buildOpts += ' skiptests';
2025                     } else {
2026                         buildOpts += " -priority=${priority}"
2027                     }
2028
2029                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2030                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2031                     // 35 characters long.
2032
2033                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2034
2035                     if (!isBuildOnly) {
2036                         def runtestArguments = ''
2037                         def testOpts = 'collectdumps'
2038
2039                         if (isR2RScenario(scenario)) {
2040
2041                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2042                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2043                             // to cause the tests to be crossgen'ed.
2044
2045                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2046                                 testOpts += ' crossgenaltjit protononjit.dll'
2047                             } else {
2048                                 testOpts += ' crossgen'
2049                             }
2050
2051                             testOpts += ' runcrossgentests'
2052                         }
2053                         else if (scenario == 'jitdiff') {
2054                             testOpts += ' jitdisasm crossgen'
2055                         }
2056                         else if (scenario == 'ilrt') {
2057                             testOpts += ' ilasmroundtrip'
2058                         }
2059                         else if (isLongGc(scenario)) {
2060                             testOpts += " ${scenario} sequential"
2061                         }
2062                         else if (scenario == 'standalone_gc') {
2063                             testOpts += ' gcname clrgc.dll'
2064                         }
2065                         else if (scenario == 'illink') {
2066                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2067                         }
2068
2069                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2070                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2071                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2072                         // which is read by the xunit harness.)
2073                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2074                         {
2075                             def timeout = 1800000
2076                             testOpts += " timeout ${timeout}"
2077                         }
2078
2079                         // If we are running a stress mode, we should write out the set of key
2080                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2081
2082                         def envScriptPath = ''
2083                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2084                             def buildCommandsStr = ''
2085                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2086                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2087
2088                             if (isJitStressScenario(scenario)) {
2089                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2090                             }
2091                             else if (isR2RStressScenario(scenario)) {
2092                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2093                             }
2094
2095                             if (architecture == 'x86_arm_altjit') {
2096                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2097                             }
2098                             else if (architecture == 'x64_arm64_altjit') {
2099                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2100                             }
2101
2102                             envScriptFinalize(os, envScriptPath)
2103
2104                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2105                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2106                             // into a single script.
2107                             buildCommands += buildCommandsStr
2108                         }
2109                         else if (architecture == 'x86_arm_altjit') {
2110                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2111                         }
2112                         else if (architecture == 'x64_arm64_altjit') {
2113                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2114                         }
2115                         if (envScriptPath != '') {
2116                             testOpts += " TestEnv ${envScriptPath}"
2117                         }
2118
2119                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2120
2121                         if (doCoreFxTesting) {
2122                             def workspaceRelativeFxRoot = "_/fx"
2123                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2124
2125                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2126
2127                             // Archive and process (only) the test results
2128                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2129                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2130
2131                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2132                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2133                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2134                         }
2135                         else if (isGcReliabilityFramework(scenario)) {
2136                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2137                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2138                         }
2139                         else {
2140                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2141                         }
2142                     } // end if (!isBuildOnly)
2143
2144                     if (!doCoreFxTesting) {
2145                         // Run the rest of the build
2146                         // Build the mscorlib for the other OS's
2147                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2148                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2149
2150                         if (arch == 'x64') {
2151                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2152                         }
2153
2154                         // Zip up the tests directory so that we don't use so much space/time copying
2155                         // 10s of thousands of files around.
2156                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2157
2158                         if (!isJitStressScenario(scenario)) {
2159                             // For Windows, pull full test results and test drops for x86/x64.
2160                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2161                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2162                         }
2163
2164                         if (scenario == 'jitdiff') {
2165                             // retrieve jit-dasm output for base commit, and run jit-diff
2166                             if (!isBuildOnly) {
2167                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2168                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2169                             }
2170                         }
2171
2172                         if (!isBuildOnly) {
2173                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2174                         }
2175                     }
2176                     break
2177                 case 'armlb':
2178                 case 'arm':
2179                     assert isArmWindowsScenario(scenario)
2180
2181                     def buildArchitecture = 'arm'
2182
2183                     def buildOpts = ''
2184
2185                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2186                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2187                     if (architecture == 'armlb') {
2188                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2189                     }
2190
2191                     if (doCoreFxTesting) {
2192                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2193                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2194                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2195                         //
2196                         // buildOpts += ' skiptests';
2197                         buildOpts += " -priority=0"
2198                     } else {
2199                         buildOpts += " -priority=${priority}"
2200                     }
2201
2202                     // This is now a build only job. Do not run tests. Use the flow job.
2203                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2204
2205                     if (doCoreFxTesting) {
2206                         assert isBuildOnly
2207                         assert architecture == 'arm'
2208
2209                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2210                         // to determine the correct matching corefx version git commit hash.
2211                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2212
2213                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2214                         def envScriptPath = ''
2215                         def buildCommandsStr = ''
2216                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2217                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2218                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2219                         envScriptFinalize(os, envScriptPath)
2220                         buildCommands += buildCommandsStr
2221
2222                         def workspaceRelativeFxRootLinux = "_/fx"
2223                         def workspaceRelativeFxRootWin = "_\\fx"
2224                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2225
2226                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2227
2228                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2229                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2230                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2231
2232                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2233                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2234                     } else {
2235                         // Zip up the tests directory so that we don't use so much space/time copying
2236                         // 10s of thousands of files around.
2237                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2238
2239                         // Add archival.
2240                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2241                     }
2242                     break
2243                 case 'arm64':
2244                     assert isArmWindowsScenario(scenario)
2245
2246                     // This is now a build only job. Do not run tests. Use the flow job.
2247                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2248
2249                     // Zip up the tests directory so that we don't use so much space/time copying
2250                     // 10s of thousands of files around.
2251                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2252
2253                     // Add archival.
2254                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2255                     break
2256                 default:
2257                     println("Unknown architecture: ${architecture}");
2258                     assert false
2259                     break
2260             }
2261             break
2262         // end case 'Windows_NT'; editor brace matching: }
2263         case 'Ubuntu':
2264         case 'Ubuntu16.04':
2265         case 'Ubuntu16.10':
2266         case 'Debian8.4':
2267         case 'OSX10.12':
2268         case 'CentOS7.1':
2269         case 'RHEL7.2':
2270         case 'Tizen':
2271         case 'Fedora24': // editor brace matching: {
2272             switch (architecture) {
2273                 case 'x64':
2274                 case 'x86':
2275                     if (architecture == 'x86' && os == 'Ubuntu') {
2276                         // build and PAL test
2277                         def dockerImage = getDockerImageName(architecture, os, true)
2278                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2279                         dockerImage = getDockerImageName(architecture, os, false)
2280                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2281                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2282                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2283                         break
2284                     }
2285
2286                     if (scenario == 'formatting') {
2287                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2288                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2289                         break
2290                     }
2291
2292                     if (scenario == 'illink') {
2293                         assert(os == 'Ubuntu')
2294                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2295                     }
2296
2297                     if (!doCoreFxTesting) {
2298                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2299                         // only on supported OS platforms.
2300                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2301                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2302
2303                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2304                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2305
2306                         // Basic archiving of the build
2307                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2308                         // And pal tests
2309                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2310                     }
2311                     else {
2312                         // Corefx stress testing
2313                         assert os == 'Ubuntu'
2314                         assert architecture == 'x64'
2315                         assert lowerConfiguration == 'checked'
2316                         assert isJitStressScenario(scenario)
2317
2318                         // Build coreclr
2319                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2320
2321                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2322
2323                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2324                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2325                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2326                         buildCommands += envScriptCmds
2327
2328                         // Build and text corefx
2329                         def workspaceRelativeFxRoot = "_/fx"
2330                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2331
2332                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2333
2334                         // Archive and process (only) the test results
2335                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2336                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2337                     }
2338                     break
2339                 case 'arm64':
2340                     if (!doCoreFxTesting) {
2341                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2342
2343                         // HACK -- Arm64 does not have corefx jobs yet.
2344                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2345                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2346                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2347                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2348
2349                         // Basic archiving of the build
2350                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2351                     }
2352                     break
2353                 case 'armem':
2354                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2355                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2356
2357                     // default values for Ubuntu
2358                     def arm_abi = "arm"
2359                     def linuxCodeName = "trusty"
2360                     if (os == 'Ubuntu16.04') {
2361                         linuxCodeName = "xenial"
2362                     }
2363                     else if (os == 'Tizen') {
2364                         arm_abi = "armel"
2365                         linuxCodeName = "tizen"
2366                     }
2367
2368                     // Unzip the Windows test binaries first. Exit with 0
2369                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2370
2371                     // Unpack the corefx binaries
2372                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2373                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2374                     if (os != 'Tizen') {
2375                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2376                     }
2377                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2378                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2379                     if (os == 'Ubuntu') {
2380                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2381                     }
2382
2383                     // Call the ARM CI script to cross build and test using docker
2384                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2385                     --mode=docker \\
2386                     --${arm_abi} \\
2387                     --linuxCodeName=${linuxCodeName} \\
2388                     --buildConfig=${lowerConfiguration} \\
2389                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2390                     --coreFxBinDir=./bin/CoreFxBinDir \\
2391                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2392
2393                     // Basic archiving of the build, no pal tests
2394                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2395                     break
2396                 case 'arm':
2397                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2398                     assert (os == 'Ubuntu')
2399
2400                     // Add some useful information to the log file. Ignore return codes.
2401                     buildCommands += "uname -a || true"
2402
2403                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2404                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2405
2406                     def dockerImage = getDockerImageName(architecture, os, true)
2407                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm ${dockerImage} "
2408
2409                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross"
2410
2411                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2412                     // download the appropriate CoreFX packages.
2413                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2414                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2415
2416                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2417
2418                     // ZIP up for the test job (created in the flow job code):
2419                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2420                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2421                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2422                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2423
2424                     // These commands are assumed to be run from the root of the workspace.
2425                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2426                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2427
2428                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2429                     break
2430                 default:
2431                     println("Unknown architecture: ${architecture}");
2432                     assert false
2433                     break
2434             }
2435             break
2436         // editor brace matching: }
2437         default:
2438             println("Unknown os: ${os}");
2439             assert false
2440             break
2441     } // os
2442
2443     return buildCommands
2444 }
2445
2446 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2447 // Returns true if the job should be generated.
2448 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2449 {
2450     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2451     // All other scenarios do Pri-1 testing.
2452     if (scenario == 'innerloop' && !isPR) {
2453         return false
2454     }
2455
2456     // Tizen is only supported for armem architecture
2457     if (os == 'Tizen' && architecture != 'armem') {
2458         return false
2459     }
2460
2461     // Filter based on architecture.
2462
2463     switch (architecture) {
2464         case 'arm64':
2465         case 'arm':
2466             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2467                 return false
2468             }
2469             break
2470         case 'armem':
2471             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2472                 return false
2473             }
2474             break
2475         case 'armlb':
2476             // Do not create armlb jobs
2477             return false
2478         case 'x86_arm_altjit':
2479         case 'x64_arm64_altjit':
2480             if (os != 'Windows_NT') {
2481                 return false
2482             }
2483             break
2484         case 'x86':
2485             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2486                 return false
2487             }
2488             break
2489         case 'x64':
2490             // Everything implemented
2491             break
2492         default:
2493             println("Unknown architecture: ${architecture}")
2494             assert false
2495             break
2496     }
2497
2498     // Which (Windows) build only jobs are required?
2499
2500     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2501
2502     if (isBuildOnly) {
2503         switch (architecture) {
2504             case 'arm':
2505                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2506                 if (!isCoreFxScenario(scenario)) {
2507                     return false
2508                 }
2509                 break
2510             case 'x64':
2511             case 'x86':
2512                 if (!isNormalOrInnerloop) {
2513                     return false
2514                 }
2515                 break
2516             default:
2517                 return false
2518         }
2519     }
2520
2521     // Filter based on scenario.
2522
2523     if (isJitStressScenario(scenario)) {
2524         if (configuration != 'Checked') {
2525             return false
2526         }
2527
2528         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2529         if (!isEnabledOS) {
2530             return false
2531         }
2532
2533         switch (architecture) {
2534             case 'x64':
2535             case 'x86_arm_altjit':
2536             case 'x64_arm64_altjit':
2537                 break
2538
2539             case 'x86':
2540                 // x86 ubuntu: no stress modes
2541                 if (os == 'Ubuntu') {
2542                     return false
2543                 }
2544                 break
2545
2546             case 'arm':
2547                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2548                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2549                     return false
2550                 }
2551                 break
2552
2553             default:
2554                 // arm64, armlb: stress is handled through flow jobs.
2555                 // armem: no stress jobs for ARM emulator.
2556                 return false
2557         }
2558     }
2559     else if (isR2RScenario(scenario)) {
2560         if (os != 'Windows_NT') {
2561             return false
2562         }
2563         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2564         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2565             return false
2566         }
2567     }
2568     else {
2569         // Skip scenarios
2570         switch (scenario) {
2571             case 'ilrt':
2572                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2573                 // the default scenario build
2574                 if (os != 'Windows_NT') {
2575                     return false
2576                 }
2577                 // Only x64 for now
2578                 if (architecture != 'x64') {
2579                     return false
2580                 }
2581                 // Release only
2582                 if (configuration != 'Release') {
2583                     return false
2584                 }
2585                 break
2586             case 'jitdiff':
2587                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2588                     return false
2589                 }
2590                 if (architecture != 'x64') {
2591                     return false
2592                 }
2593                 if (configuration != 'Checked') {
2594                     return false
2595                 }
2596                 break
2597             case 'longgc':
2598             case 'gcsimulator':
2599                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2600                     return false
2601                 }
2602                 if (architecture != 'x64') {
2603                     return false
2604                 }
2605                 if (configuration != 'Release') {
2606                     return false
2607                 }
2608                 break
2609             case 'gc_reliability_framework':
2610             case 'standalone_gc':
2611                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2612                     return false
2613                 }
2614
2615                 if (architecture != 'x64') {
2616                     return false
2617                 }
2618
2619                 if (configuration != 'Release' && configuration != 'Checked') {
2620                     return false
2621                 }
2622                 break
2623             // We only run Windows and Ubuntu x64 Checked for formatting right now
2624             case 'formatting':
2625                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2626                     return false
2627                 }
2628                 if (architecture != 'x64') {
2629                     return false
2630                 }
2631                 if (configuration != 'Checked') {
2632                     return false
2633                 }
2634                 break
2635             case 'illink':
2636                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2637                     return false
2638                 }
2639                 if (architecture != 'x64' && architecture != 'x86') {
2640                     return false
2641                 }
2642                 break
2643             case 'normal':
2644                 // Nothing skipped
2645                 break
2646             case 'innerloop':
2647                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2648                     return false
2649                 }
2650                 break
2651             default:
2652                 println("Unknown scenario: ${scenario}")
2653                 assert false
2654                 break
2655         }
2656     }
2657
2658     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2659     // runtime behavior, not compile-time behavior, are not interesting.
2660     switch (architecture) {
2661         case 'x86_arm_altjit':
2662         case 'x64_arm64_altjit':
2663             if (isGCStressRelatedTesting(scenario)) {
2664                 return false
2665             }
2666             break
2667         default:
2668             break
2669     }
2670
2671     // The job was not filtered out, so we should generate it!
2672     return true
2673 }
2674
2675 Constants.allScenarios.each { scenario ->
2676     [true, false].each { isPR ->
2677         Constants.architectureList.each { architecture ->
2678             Constants.configurationList.each { configuration ->
2679                 Constants.osList.each { os ->
2680                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2681                     // and reset the os to Windows_NT
2682                     def isBuildOnly = false
2683                     if (os == 'Windows_NT_BuildOnly') {
2684                         isBuildOnly = true
2685                         os = 'Windows_NT'
2686                     }
2687
2688                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2689                         return
2690                     }
2691
2692                     // Calculate names
2693                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2694                     def folderName = getJobFolder(scenario)
2695
2696                     // Create the new job
2697                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2698                     addToViews(newJob, isPR, architecture, os)
2699
2700                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2701
2702                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2703                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2704                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2705
2706                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2707                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2708                     // artifacts produced.
2709                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2710                         // Define the Windows Tests and Corefx build job names
2711                         def lowerConfiguration = configuration.toLowerCase()
2712                         def WindowsTestsName = projectFolder + '/' +
2713                                                Utilities.getFullJobName(project,
2714                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2715                                                                         false)
2716                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2717                                            Utilities.getFolderName(branch)
2718
2719                         def arm_abi = 'arm'
2720                         def corefx_os = 'linux'
2721                         if (os == 'Tizen') {
2722                             arm_abi = 'armel'
2723                             corefx_os = 'tizen'
2724                         }
2725
2726                         // Let's use release CoreFX to test checked CoreCLR,
2727                         // because we do not generate checked CoreFX in CoreFX CI yet.
2728                         def corefx_lowerConfiguration = lowerConfiguration
2729                         if (lowerConfiguration == 'checked') {
2730                             corefx_lowerConfiguration = 'release'
2731                         }
2732
2733                         // Copy the Windows test binaries and the Corefx build binaries
2734                         newJob.with {
2735                             steps {
2736                                 copyArtifacts(WindowsTestsName) {
2737                                     includePatterns('bin/tests/tests.zip')
2738                                     buildSelector {
2739                                         latestSuccessful(true)
2740                                     }
2741                                 }
2742                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2743                                     includePatterns('bin/build.tar.gz')
2744                                     buildSelector {
2745                                         latestSuccessful(true)
2746                                     }
2747                                 }
2748                             } // steps
2749                         } // newJob.with
2750                     }
2751
2752                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2753
2754                     newJob.with {
2755                         steps {
2756                             if (os == 'Windows_NT') {
2757                                 buildCommands.each { buildCommand ->
2758                                     batchFile(buildCommand)
2759                                 }
2760                             }
2761                             else {
2762                                 buildCommands.each { buildCommand ->
2763                                     shell(buildCommand)
2764                                 }
2765                             }
2766                         } // steps
2767                     } // newJob.with
2768
2769                 } // os
2770             } // configuration
2771         } // architecture
2772     } // isPR
2773 } // scenario
2774
2775 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2776 // Returns the newly created job.
2777 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2778 {
2779     def osGroup = getOSGroup(os)
2780     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2781
2782     def jobFolder = getJobFolder(scenario)
2783     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2784         parameters {
2785             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2786         }
2787
2788         steps {
2789             // Set up the copies
2790
2791             // Coreclr build we are trying to test
2792             //
2793             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2794
2795             copyArtifacts(inputCoreCLRBuildName) {
2796                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2797                 buildSelector {
2798                     buildNumber('${CORECLR_BUILD}')
2799                 }
2800             }
2801
2802             if (isCoreFxScenario(scenario)) {
2803
2804                 // Only arm supported for corefx testing now.
2805                 assert architecture == 'arm'
2806
2807                 // Unzip CoreFx runtime
2808                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2809
2810                 // Unzip CoreFx tests.
2811                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2812
2813                 // Add the script to run the corefx tests
2814                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2815                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2816                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2817                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2818
2819             } else { // !isCoreFxScenario(scenario)
2820
2821                 // Unzip tests.
2822                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2823
2824                 def buildCommands = ""
2825
2826                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2827                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2828                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2829
2830                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2831                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2832
2833                 // For all jobs
2834                 addEnvVariable("CORE_ROOT", coreRootLocation)
2835                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2836                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2837
2838                 // ARM legacy backend; this is an altjit.
2839                 if (architecture == 'armlb') {
2840                     addEnvVariable("COMPlus_AltJit", "*")
2841                     addEnvVariable("COMPlus_AltJitNgen", "*")
2842                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2843                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2844                 }
2845
2846                 // If we are running a stress mode, we'll set those variables as well
2847                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2848                     def stressValues = null
2849                     if (isJitStressScenario(scenario)) {
2850                         stressValues = Constants.jitStressModeScenarios[scenario]
2851                     }
2852                     else {
2853                         stressValues = Constants.r2rStressScenarios[scenario]
2854                     }
2855
2856                     stressValues.each { key, value ->
2857                         addEnvVariable(key, value)
2858                     }
2859                 }
2860
2861                 if (isR2RScenario(scenario)) {
2862                     // Crossgen the framework assemblies.
2863                     buildCommands += """
2864 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2865 @goto skip_PrecompileAssembly
2866
2867 :PrecompileAssembly
2868 @REM Skip mscorlib since it is already precompiled.
2869 @if /I "%3" == "mscorlib.dll" exit /b 0
2870 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2871
2872 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2873 @if "%errorlevel%" == "-2146230517" (
2874     echo %2 is not a managed assembly.
2875 ) else if "%errorlevel%" == "-2146234344" (
2876     echo %2 is not a managed assembly.
2877 ) else if %errorlevel% neq 0 (
2878     echo Unable to precompile %2
2879 ) else (
2880     echo Precompiled %2
2881 )
2882 @exit /b 0
2883
2884 :skip_PrecompileAssembly
2885 """
2886
2887                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2888                     // crossgen on tests before running them.
2889                     addEnvVariable("RunCrossGen", "true")
2890                 } // isR2RScenario(scenario)
2891
2892                 // Create the smarty command
2893                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2894                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2895                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2896                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2897
2898                 if (architecture == 'armlb') {
2899                     addExclude("LEGACYJIT_FAIL")
2900                 }
2901
2902                 // Exclude tests based on scenario.
2903                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2904                     addArchSpecificExclude(architecture, excludeTag)
2905                 }
2906
2907                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2908                 if (scenario == 'innerloop') {
2909                     addExclude("pri1")
2910                 }
2911
2912                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2913                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2914                 addExclude("LONG_RUNNING")
2915
2916                 smartyCommand += "/lstFile Tests.lst"
2917
2918                 def testListArch = [
2919                     'arm64': 'arm64',
2920                     'arm': 'arm',
2921                     'armlb': 'arm'
2922                 ]
2923
2924                 def archLocation = testListArch[architecture]
2925
2926                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2927                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2928                 addCommand("${smartyCommand}")
2929
2930                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2931                 // However, we also need to remove all the variables that were set during this batch file, so we
2932                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2933                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2934                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2935                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2936                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2937                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2938
2939                 // ZIP up the smarty output, no matter what the smarty result.
2940                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2941
2942                 addCommand("echo %errorlevel%")
2943                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2944
2945                 // Use the smarty errorlevel as the script errorlevel.
2946                 addCommand("exit /b %__save_smarty_errorlevel%")
2947
2948                 batchFile(buildCommands)
2949             } // non-corefx testing
2950         } // steps
2951     } // job
2952
2953     if (!isCoreFxScenario(scenario)) {
2954         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2955
2956         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2957         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2958         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2959         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2960     }
2961
2962     return newJob
2963 }
2964
2965 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2966 // E.g., non-Windows tests.
2967 // Returns the newly created job.
2968 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2969 {
2970     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2971
2972     def osGroup = getOSGroup(os)
2973     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2974
2975     def testOpts = ''
2976     def useServerGC = false
2977
2978     // Enable Server GC for Ubuntu PR builds
2979     // REVIEW: why? Does this apply to all architectures? Why only PR?
2980     if (os == 'Ubuntu' && isPR) {
2981         testOpts += ' --useServerGC'
2982         useServerGC = true
2983     }
2984
2985     if (isR2RScenario(scenario)) {
2986
2987         testOpts += ' --crossgen --runcrossgentests'
2988
2989         if (scenario == 'r2r_jitstress1') {
2990             testOpts += ' --jitstress=1'
2991         }
2992         else if (scenario == 'r2r_jitstress2') {
2993             testOpts += ' --jitstress=2'
2994         }
2995         else if (scenario == 'r2r_jitstressregs1') {
2996             testOpts += ' --jitstressregs=1'
2997         }
2998         else if (scenario == 'r2r_jitstressregs2') {
2999             testOpts += ' --jitstressregs=2'
3000         }
3001         else if (scenario == 'r2r_jitstressregs3') {
3002             testOpts += ' --jitstressregs=3'
3003         }
3004         else if (scenario == 'r2r_jitstressregs4') {
3005             testOpts += ' --jitstressregs=4'
3006         }
3007         else if (scenario == 'r2r_jitstressregs8') {
3008             testOpts += ' --jitstressregs=8'
3009         }
3010         else if (scenario == 'r2r_jitstressregs0x10') {
3011             testOpts += ' --jitstressregs=0x10'
3012         }
3013         else if (scenario == 'r2r_jitstressregs0x80') {
3014             testOpts += ' --jitstressregs=0x80'
3015         }
3016         else if (scenario == 'r2r_jitstressregs0x1000') {
3017             testOpts += ' --jitstressregs=0x1000'
3018         }
3019         else if (scenario == 'r2r_jitminopts') {
3020             testOpts += ' --jitminopts'
3021         }
3022         else if (scenario == 'r2r_jitforcerelocs') {
3023             testOpts += ' --jitforcerelocs'
3024         }
3025         else if (scenario == 'r2r_gcstress15') {
3026             testOpts += ' --gcstresslevel=0xF'
3027         }
3028     }
3029     else if (scenario == 'jitdiff') {
3030         testOpts += ' --jitdisasm --crossgen'
3031     }
3032     else if (scenario == 'illink') {
3033         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3034     }
3035     else if (isLongGc(scenario)) {
3036         // Long GC tests behave very poorly when they are not
3037         // the only test running (many of them allocate until OOM).
3038         testOpts += ' --sequential'
3039
3040         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3041         // for running long GC and GCSimulator tests, respectively. We don't use them
3042         // here because using a playlist file produces much more readable output on the CI machines
3043         // and reduces running time.
3044         //
3045         // The Long GC playlist contains all of the tests that are
3046         // going to be run. The GCSimulator playlist contains all of
3047         // the GC simulator tests.
3048         if (scenario == 'longgc') {
3049             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3050         }
3051         else if (scenario == 'gcsimulator') {
3052             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3053         }
3054     }
3055     else if (isGcReliabilityFramework(scenario)) {
3056         testOpts += ' --build-overlay-only'
3057     }
3058     else if (scenario == 'standalone_gc') {
3059         if (osGroup == 'OSX') {
3060             testOpts += ' --gcname=libclrgc.dylib'
3061         }
3062         else if (osGroup == 'Linux') {
3063             testOpts += ' --gcname=libclrgc.so'
3064         }
3065         else {
3066             println("Unexpected OS group: ${osGroup} for os ${os}")
3067             assert false
3068         }
3069     }
3070
3071     def jobFolder = getJobFolder(scenario)
3072     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3073         parameters {
3074             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3075             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3076         }
3077
3078         steps {
3079             // Set up the copies
3080
3081             // Coreclr build containing the tests and mscorlib
3082             // pri1 jobs still need to copy windows_nt built tests
3083             assert inputTestsBuildName != null
3084             copyArtifacts(inputTestsBuildName) {
3085                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3086                 buildSelector {
3087                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3088                 }
3089             }
3090
3091             // Coreclr build we are trying to test
3092             //
3093             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3094
3095             copyArtifacts(inputCoreCLRBuildName) {
3096                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3097                 buildSelector {
3098                     buildNumber('${CORECLR_BUILD}')
3099                 }
3100             }
3101
3102             if (isUbuntuArmJob) {
3103                 // Add some useful information to the log file. Ignore return codes.
3104                 shell("uname -a || true")
3105             }
3106
3107             if (architecture == 'arm64') {
3108                 shell("mkdir -p ./bin/CoreFxBinDir")
3109                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3110                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3111             }
3112             else if (architecture == 'x86') {
3113                 shell("mkdir ./bin/CoreFxNative")
3114
3115                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3116
3117                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3118                     includePatterns('bin/build.tar.gz')
3119                     targetDirectory('bin/CoreFxNative')
3120                     buildSelector {
3121                         latestSuccessful(true)
3122                     }
3123                 }
3124
3125                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3126             }
3127
3128             // Unzip the tests first.  Exit with 0
3129             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3130             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3131
3132             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3133             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3134             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3135             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3136             // copied correctly.
3137             if (isUbuntuArmJob) {
3138                 def lowerConfiguration = configuration.toLowerCase()
3139                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3140                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3141             }
3142             else {
3143                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3144             }
3145
3146             // Execute the tests
3147             def runDocker = isNeedDocker(architecture, os, false)
3148             def dockerPrefix = ""
3149             def dockerCmd = ""
3150             if (runDocker) {
3151                 def dockerImage = getDockerImageName(architecture, os, false)
3152                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3153                 dockerCmd = dockerPrefix + "${dockerImage} "
3154             }
3155
3156             // If we are running a stress mode, we'll set those variables first
3157             if (isJitStressScenario(scenario)) {
3158                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3159                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3160                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3161                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3162                 shell("${envScriptCmds}")
3163                 testOpts += " --test-env=${scriptFileName}"
3164             }
3165
3166             // TODO: how to handle GCStress-related testing for Ubuntu/arm?
3167             if (isGCStressRelatedTesting(scenario)) {
3168                 shell('./init-tools.sh')
3169             }
3170
3171             def runScript = ""
3172             if (isUbuntuArmJob) {
3173                 // Use 'runtesttilstable.sh' to rerun failing tests (in sequential mode);
3174                 // there are many tests that pass on rerun (currently), and we don't want
3175                 // that flakiness to affect overall test job robustness.
3176                 runScript = "${dockerCmd}./tests/runtesttilstable.sh"
3177             } else {
3178                 runScript = "${dockerCmd}./tests/runtest.sh"
3179             }
3180
3181             shell("""\
3182 ${runScript} \\
3183     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3184     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3185     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3186     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3187
3188             if (isGcReliabilityFramework(scenario)) {
3189                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3190                 if (useServerGC) {
3191                     if (runDocker) {
3192                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3193                     }
3194                     else {
3195                         shell("export COMPlus_gcServer=1")
3196                     }
3197                 }
3198
3199                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3200             }
3201         } // steps
3202     } // job
3203
3204     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3205     if (os in ['Ubuntu']) {
3206         SummaryBuilder summaries = new SummaryBuilder()
3207         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3208         summaries.emit(newJob)
3209     }
3210
3211     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3212     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3213
3214     return newJob
3215 }
3216
3217 // Create a test job that will be used by a flow job.
3218 // Returns the newly created job.
3219 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3220 {
3221     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3222
3223     def newJob = null
3224     if (windowsArmJob) {
3225         assert inputTestsBuildName == null
3226         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3227     } else {
3228         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3229     }
3230
3231     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3232
3233     addToViews(newJob, isPR, architecture, os)
3234
3235     if (scenario == 'jitdiff') {
3236         def osGroup = getOSGroup(os)
3237         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3238     }
3239
3240     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3241     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3242
3243     return newJob
3244 }
3245
3246 // Create a flow job to tie together a build job with the given test job.
3247 // Returns the new flow job.
3248 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3249 {
3250     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3251     // Linux CoreCLR test
3252     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3253     def jobFolder = getJobFolder(scenario)
3254
3255     def newFlowJob = null
3256
3257     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3258     if (windowsArmJob) {
3259
3260         assert inputTestsBuildName == null
3261
3262         // For Windows arm jobs there is no reason to build a parallel test job.
3263         // The product build supports building and archiving the tests.
3264
3265         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3266                         buildFlow("""\
3267 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3268
3269 // And then build the test build
3270 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3271 """)
3272         }
3273         JobReport.Report.addReference(inputCoreCLRBuildName)
3274         JobReport.Report.addReference(fullTestJobName)
3275     }
3276     else {
3277         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3278                         buildFlow("""\
3279 // Build the input jobs in parallel
3280 parallel (
3281 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3282 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3283 )
3284
3285 // And then build the test build
3286 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3287                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3288 """)
3289         }
3290         JobReport.Report.addReference(inputCoreCLRBuildName)
3291         JobReport.Report.addReference(inputTestsBuildName)
3292         JobReport.Report.addReference(fullTestJobName)
3293     }
3294
3295     addToViews(newFlowJob, isPR, architecture, os)
3296
3297     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3298
3299     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3300     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3301
3302     return newFlowJob
3303 }
3304
3305 // Determine if we should generate a flow job for the given parameters.
3306 // Returns true if the job should be generated.
3307 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3308 {
3309     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3310     // All other scenarios do Pri-1 testing.
3311     if (scenario == 'innerloop' && !isPR) {
3312         return false
3313     }
3314
3315     // Filter based on OS and architecture.
3316
3317     switch (architecture) {
3318         case 'arm64':
3319             if (os != "Ubuntu" && os != "Windows_NT") {
3320                 return false
3321             }
3322             break
3323         case 'armlb':
3324             if (os != 'Windows_NT') {
3325                 return false
3326             }
3327             // Do not create armlb windows jobs.
3328             return false
3329         case 'arm':
3330             if (os != "Ubuntu" && os != "Windows_NT") {
3331                 return false
3332             }
3333             break
3334         case 'x86':
3335             if (os != "Ubuntu") {
3336                 return false
3337             }
3338             break
3339         case 'x64':
3340             if (!(os in Constants.crossList)) {
3341                 return false
3342             }
3343             if (os == "Windows_NT") {
3344                 return false
3345             }
3346             break
3347         case 'armem':
3348         case 'x86_arm_altjit':
3349         case 'x64_arm64_altjit':
3350             // No flow jobs
3351             return false
3352         default:
3353             println("Unknown architecture: ${architecture}")
3354             assert false
3355             break
3356     }
3357
3358     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3359
3360     // Filter based on scenario in OS.
3361
3362     if (os == 'Windows_NT') {
3363         if (!isArmWindowsScenario(scenario)) {
3364             return false
3365         }
3366     }
3367     else {
3368         // Non-Windows
3369         if (architecture == 'arm64') {
3370             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3371                 return false
3372             }
3373         }
3374         else if (architecture == 'arm') {
3375             if (!(scenario in Constants.validLinuxArmScenarios)) {
3376                 return false
3377             }
3378         }
3379         else if (architecture == 'x86') {
3380             // Linux/x86 only want innerloop and default test
3381             if (!isNormalOrInnerloop) {
3382                 return false
3383             }
3384         }
3385     }
3386
3387     // For CentOS, we only want Checked/Release builds.
3388     if (os == 'CentOS7.1') {
3389         if (configuration != 'Checked' && configuration != 'Release') {
3390             return false
3391         }
3392         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3393             return false
3394         }
3395     }
3396
3397     // For RedHat and Debian, we only do Release builds.
3398     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3399         if (configuration != 'Release') {
3400             return false
3401         }
3402         if (!isNormalOrInnerloop) {
3403             return false
3404         }
3405     }
3406
3407     // Next, filter based on scenario.
3408
3409     if (isJitStressScenario(scenario)) {
3410         if (configuration != 'Checked') {
3411             return false
3412         }
3413
3414         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3415         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3416             return false
3417         }
3418     }
3419     else if (isR2RBaselineScenario(scenario)) {
3420         if (configuration != 'Checked' && configuration != 'Release') {
3421             return false
3422         }
3423     }
3424     else if (isR2RStressScenario(scenario)) {
3425         if (configuration != 'Checked') {
3426             return false
3427         }
3428     }
3429     else {
3430         // Skip scenarios
3431         switch (scenario) {
3432             case 'ilrt':
3433             case 'longgc':
3434             case 'gcsimulator':
3435                 // Long GC tests take a long time on non-Release builds
3436                 // ilrt is also Release only
3437                 if (configuration != 'Release') {
3438                     return false
3439                 }
3440                 break
3441
3442             case 'jitdiff':
3443                 if (configuration != 'Checked') {
3444                     return false
3445                 }
3446                 break
3447
3448             case 'gc_reliability_framework':
3449             case 'standalone_gc':
3450                 if (configuration != 'Release' && configuration != 'Checked') {
3451                     return false
3452                 }
3453                 break
3454
3455             case 'formatting':
3456                 return false
3457             case 'illink':
3458                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3459                     return false
3460                 }
3461                 break
3462
3463             case 'normal':
3464                 // Nothing skipped
3465                 break
3466
3467             case 'innerloop':
3468                 // Nothing skipped
3469                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3470                     return false
3471                 }
3472                 break
3473
3474             default:
3475                 println("Unknown scenario: ${scenario}")
3476                 assert false
3477                 break
3478         }
3479     }
3480
3481     // The job was not filtered out, so we should generate it!
3482     return true
3483 }
3484
3485 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3486 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3487 Constants.allScenarios.each { scenario ->
3488     [true, false].each { isPR ->
3489         Constants.architectureList.each { architecture ->
3490             Constants.configurationList.each { configuration ->
3491                 Constants.osList.each { os ->
3492
3493                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3494                         return
3495                     }
3496
3497                     // Figure out the job name of the CoreCLR build the test will depend on.
3498
3499                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3500                     def inputCoreCLRBuildIsBuildOnly = false
3501                     if (isCoreFxScenario(scenario)) {
3502                         // Every CoreFx test depends on its own unique build.
3503                         inputCoreCLRBuildScenario = scenario
3504                         inputCoreCLRBuildIsBuildOnly = true
3505                     }
3506                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3507                     def inputCoreCLRBuildName = projectFolder + '/' +
3508                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3509
3510                     // Figure out the name of the build job that the test job will depend on.
3511                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3512                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3513
3514                     def inputTestsBuildName = null
3515
3516                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3517                     if (!windowsArmJob) {
3518                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3519
3520                         def inputTestsBuildArch = architecture
3521                         if (architecture == "arm64") {
3522                             // Use the x64 test build for arm64 unix
3523                             inputTestsBuildArch = "x64"
3524                         }
3525                         else if (architecture == "arm") {
3526                             // Use the x86 test build for arm unix
3527                             inputTestsBuildArch = "x86"
3528                         }
3529
3530                         def inputTestsBuildIsBuildOnly = true
3531
3532                         inputTestsBuildName = projectFolder + '/' +
3533                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3534                     }
3535
3536                     // =============================================================================================
3537                     // Create the test job
3538                     // =============================================================================================
3539
3540                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3541
3542                     // =============================================================================================
3543                     // Create a build flow to join together the build and tests required to run this test.
3544                     // =============================================================================================
3545
3546                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3547                         // Do not create the flow job for RHEL jobs.
3548                         return
3549                     }
3550
3551                     def fullTestJobName = projectFolder + '/' + testJob.name
3552                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3553
3554                 } // os
3555             } // configuration
3556         } // architecture
3557     } // isPR
3558 } // scenario
3559
3560 JobReport.Report.generateJobReport(out)
3561
3562 // Make the call to generate the help job
3563 Utilities.createHelperJob(this, project, branch,
3564     "Welcome to the ${project} Repository",  // This is prepended to the help message
3565     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3566
3567 Utilities.addCROSSCheck(this, project, branch)