netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213     ]
 214
 215     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 216     // to Tests.lst file categories to exclude.
 217     //
 218     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 219     // and with the same values, with some commented out, for easier maintenance.
 220     //
 221     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 222     //
 223     def static validArmWindowsScenarios = [
 224                'innerloop':                              [],
 225                'normal':                                 [],
 226                // 'ilrt'
 227                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 228                // 'longgc'
 229                // 'formatting'
 230                // 'gcsimulator'
 231                // 'jitdiff'
 232                // 'standalone_gc'
 233                // 'gc_reliability_framework'
 234                // 'illink'
 235                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 246                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 248                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 249                'tieredcompilation':                      [],
 250                'forcerelocs':                            [],
 251                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 270                // 'jitsse2only'                          // Only relevant to xarch
 271                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 272                // 'jitincompletehwintrinsic'
 273                // 'jitx86hwintrinsicnoavx'
 274                // 'jitx86hwintrinsicnoavx2'
 275                // 'jitx86hwintrinsicnosimd'
 276                // 'jitnox86hwintrinsic'
 277                'corefx_baseline':                        [],    // corefx tests don't use smarty
 278                'corefx_minopts':                         [],    // corefx tests don't use smarty
 279                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 280                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 281                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 283                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 284                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 285                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 286                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 290                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 291                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 292                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 293                'heapverify1':                            [],
 294                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 295                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 297                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 300
 301                //
 302                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 303                //
 304
 305                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 306                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 324     ]
 325
 326     def static validLinuxArm64Scenarios = [
 327                'innerloop',
 328                'normal',
 329                'r2r',
 330                'gcstress0x3',
 331                'gcstress0xc'
 332     ]
 333
 334     // Note: no GCStress-related scenario is enabled currently.
 335     def static validLinuxArmScenarios = [
 336                'innerloop',
 337                'normal',
 338                'r2r',
 339                'r2r_jitstress1',
 340                'r2r_jitstress2',
 341                'r2r_jitstressregs1',
 342                'r2r_jitstressregs2',
 343                'r2r_jitstressregs3',
 344                'r2r_jitstressregs4',
 345                'r2r_jitstressregs8',
 346                'r2r_jitstressregs0x10',
 347                'r2r_jitstressregs0x80',
 348                'r2r_jitstressregs0x1000',
 349                'r2r_jitminopts',
 350                'r2r_jitforcerelocs',
 351                // 'r2r_gcstress15',
 352                'minopts',
 353                'forcerelocs',
 354                'jitstress1',
 355                'jitstress2',
 356                'jitstressregs1',
 357                'jitstressregs2',
 358                'jitstressregs3',
 359                'jitstressregs4',
 360                'jitstressregs8',
 361                'jitstressregs0x10',
 362                'jitstressregs0x80',
 363                'jitstressregs0x1000',
 364                'jitstress2_jitstressregs1',
 365                'jitstress2_jitstressregs2',
 366                'jitstress2_jitstressregs3',
 367                'jitstress2_jitstressregs4',
 368                'jitstress2_jitstressregs8',
 369                'jitstress2_jitstressregs0x10',
 370                'jitstress2_jitstressregs0x80',
 371                'jitstress2_jitstressregs0x1000',
 372                'tailcallstress'
 373                // 'gcstress0x3',
 374                // 'gcstress0xc',
 375                // 'zapdisable',
 376                // 'heapverify1',
 377                // 'gcstress0xc_zapdisable',
 378                // 'gcstress0xc_zapdisable_jitstress2',
 379                // 'gcstress0xc_zapdisable_heapverify1',
 380                // 'gcstress0xc_jitstress1',
 381                // 'gcstress0xc_jitstress2',
 382                // 'gcstress0xc_minopts_heapverify1'
 383     ]
 384
 385     def static configurationList = ['Debug', 'Checked', 'Release']
 386
 387     // This is the set of architectures
 388     // Some of these are pseudo-architectures:
 389     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 390     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 391     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 392     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 393     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 394
 395     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 396     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 397 }
 398
 399 // **************************************************************
 400 // Create some specific views
 401 //
 402 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 403 // views based on a single regular expression. These views will be generated by adding a
 404 // specific set of jobs to them.
 405 //
 406 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 407 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 408 // those are required, don't add them (which simplifies the view pages, as well).
 409 // **************************************************************
 410
 411 class Views {
 412     def static MergeJobView = null
 413     def static PeriodicJobView = null
 414     def static ArchitectureViews = [:]
 415     def static OSViews = [:]
 416 }
 417
 418 // MergeJobView: include all jobs that execute when a PR change is merged.
 419 Views.MergeJobView = listView('Merge') {
 420     recurse()
 421     columns {
 422         status()
 423         weather()
 424         name()
 425         lastSuccess()
 426         lastFailure()
 427         lastDuration()
 428         buildButton()
 429     }
 430 }
 431
 432 // PeriodicJobView: include all jobs that execute on a schedule
 433 Views.PeriodicJobView = listView('Periodic') {
 434     recurse()
 435     columns {
 436         status()
 437         weather()
 438         name()
 439         lastSuccess()
 440         lastFailure()
 441         lastDuration()
 442         buildButton()
 443     }
 444 }
 445
 446 // Create a view for non-PR jobs for each architecture.
 447 Constants.architectureList.each { architecture ->
 448     Views.ArchitectureViews[architecture] = listView(architecture) {
 449         recurse()
 450         columns {
 451             status()
 452             weather()
 453             name()
 454             lastSuccess()
 455             lastFailure()
 456             lastDuration()
 457             buildButton()
 458         }
 459     }
 460 }
 461
 462 // Create a view for non-PR jobs for each OS.
 463 Constants.osList.each { os ->
 464     // Don't create one for the special 'Windows_NT_BuildOnly'
 465     if (os == 'Windows_NT_BuildOnly') {
 466         return
 467     }
 468     Views.OSViews[os] = listView(os) {
 469         recurse()
 470         columns {
 471             status()
 472             weather()
 473             name()
 474             lastSuccess()
 475             lastFailure()
 476             lastDuration()
 477             buildButton()
 478         }
 479     }
 480 }
 481
 482 def static addToMergeView(def job) {
 483     Views.MergeJobView.with {
 484         jobs {
 485             name(job.name)
 486         }
 487     }
 488 }
 489
 490 def static addToPeriodicView(def job) {
 491     Views.PeriodicJobView.with {
 492         jobs {
 493             name(job.name)
 494         }
 495     }
 496 }
 497
 498 def static addToViews(def job, def isPR, def architecture, def os) {
 499     if (isPR) {
 500         // No views want PR jobs currently.
 501         return
 502     }
 503
 504     // Add to architecture view.
 505     Views.ArchitectureViews[architecture].with {
 506         jobs {
 507             name(job.name)
 508         }
 509     }
 510
 511     // Add to OS view.
 512     Views.OSViews[os].with {
 513         jobs {
 514             name(job.name)
 515         }
 516     }
 517 }
 518
 519 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 520     addToPeriodicView(job)
 521     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 522 }
 523
 524 def static addGithubPushTriggerHelper(def job) {
 525     addToMergeView(job)
 526     Utilities.addGithubPushTrigger(job)
 527 }
 528
 529
 530 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 531     assert os instanceof String
 532     assert architecture instanceof String
 533
 534     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 535     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 536
 537     if (!(architecture in armArches)) {
 538         assert options == null
 539         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 540
 541         return
 542     }
 543
 544     // This is an arm(64) job.
 545     //
 546     // There are several options.
 547     //
 548     // Windows_NT
 549     //
 550     // Arm32 (Build) -> latest-arm64
 551     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 552     // Arm32 (Test)  -> arm64-windows_nt
 553     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 554     //
 555     // Arm64 (Build) -> latest-arm64
 556     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 557     // Arm64 (Test)  -> arm64-windows_nt
 558     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 559     //
 560     // Ubuntu
 561     //
 562     // Arm32 emulator (Build, Test) -> arm-cross-latest
 563     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 564     //
 565     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 566     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 567     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 568     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 569     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 570     //       |-> os == "Ubuntu" && (architecture == "arm")
 571     //
 572     // Arm64 (Build) -> arm64-cross-latest
 573     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 574     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 575     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 576     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 577     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 578
 579     // This has to be a arm arch
 580     assert architecture in armArches
 581     if (os == "Windows_NT") {
 582         // arm32/arm64 Windows jobs share the same machines for now
 583         def isBuild = options['use_arm64_build_machine'] == true
 584
 585         if (isBuild == true) {
 586             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 587         } else {
 588             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 589         }
 590     } else {
 591         assert os != 'Windows_NT'
 592         assert os in supportedArmLinuxOs
 593
 594         if (architecture == 'arm64') {
 595             if ((options != null) && (options['is_build_only'] == true)) {
 596                 // Arm64 Linux build machine
 597                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 598             } else {
 599                 // Arm64 Linux test machines
 600                 if ((options != null) && (options['large_pages'] == true)) {
 601                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 602                 } else {
 603                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 604                 }
 605             }
 606         }
 607         else if (architecture == 'armem') {
 608             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 609             // using Docker.
 610             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 611         }
 612         else {
 613             // arm Ubuntu on hardware.
 614             assert (architecture == 'arm') && (os == 'Ubuntu')
 615             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 616             def isBuild = (options != null) && (options['is_build_job'] == true)
 617             if (isFlow || isBuild) {
 618                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 619                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 620                 // arm hardware.
 621                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 622             } else {
 623                 // arm Ubuntu test machine
 624                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 625                 // Utilities.setMachineAffinity. Just add the machine affinity
 626                 // manually. We specify the Helix queue name here.
 627                 job.with {
 628                     label('ubuntu.1404.arm32.open')
 629                 }
 630             }
 631         }
 632     }
 633 }
 634
 635 // setJobMachineAffinity: compute the machine affinity options for a job,
 636 // then set the job with those affinity options.
 637 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 638 {
 639     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 640            (!isBuildJob && isTestJob  && !isFlowJob) ||
 641            (!isBuildJob && !isTestJob && isFlowJob)
 642
 643     def affinityOptions = null
 644     def affinityArchitecture = architecture
 645
 646     if (os == "Windows_NT") {
 647         if (architecture in Constants.armWindowsCrossArchitectureList) {
 648             if (isBuildJob) {
 649                 affinityOptions = [ "use_arm64_build_machine" : true ]
 650             } else if (isTestJob) {
 651                 affinityOptions = [ "use_arm64_build_machine" : false ]
 652             } else if (isFlowJob) {
 653                 // For the flow jobs set the machine affinity as x64
 654                 affinityArchitecture = 'x64'
 655             }
 656         }
 657     }
 658     else {
 659         if (architecture == 'arm64') {
 660             if (isBuildJob) {
 661                 affinityOptions = ['is_build_only': true]
 662             } else if (isTestJob) {
 663                 affinityOptions = [ "large_pages" : false ]
 664             }
 665         }
 666         else if (architecture == 'arm') {
 667             if (isBuildJob) {
 668                 affinityOptions = ['is_build_job': true]
 669             } else if (isFlowJob) {
 670                 affinityOptions = ['is_flow_job': true]
 671             }
 672         }
 673     }
 674
 675     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 676 }
 677
 678 def static isGCStressRelatedTesting(def scenario) {
 679     // The 'r2r_gcstress15' scenario is a basic scenario.
 680     // Detect it and make it a GCStress related.
 681     if (scenario == 'r2r_gcstress15')
 682     {
 683         return true;
 684     }
 685
 686     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 687     def scenarioName = scenario.toLowerCase()
 688     def isGCStressTesting = false
 689     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 690         if (k in gcStressTestEnvVars) {
 691             isGCStressTesting = true;
 692         }
 693     }
 694     return isGCStressTesting
 695 }
 696
 697 def static isCoreFxScenario(def scenario) {
 698     def corefx_prefix = 'corefx_'
 699     if (scenario.length() < corefx_prefix.length()) {
 700         return false
 701     }
 702     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 703 }
 704
 705 def static isR2RBaselineScenario(def scenario) {
 706     return (scenario == 'r2r')
 707 }
 708
 709 def static isR2RStressScenario(def scenario) {
 710     return Constants.r2rStressScenarios.containsKey(scenario)
 711 }
 712
 713 def static isR2RScenario(def scenario) {
 714     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 715 }
 716
 717 def static isJitStressScenario(def scenario) {
 718     return Constants.jitStressModeScenarios.containsKey(scenario)
 719 }
 720
 721 def static isLongGc(def scenario) {
 722     return (scenario == 'longgc' || scenario == 'gcsimulator')
 723 }
 724
 725 def static isJitDiff(def scenario) {
 726     return (scenario == 'jitdiff')
 727 }
 728
 729 def static isGcReliabilityFramework(def scenario) {
 730     return (scenario == 'gc_reliability_framework')
 731 }
 732
 733 def static isArmWindowsScenario(def scenario) {
 734     return Constants.validArmWindowsScenarios.containsKey(scenario)
 735 }
 736
 737 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 738     if (isBuildOnly == true) {
 739         os = 'Windows_NT_BuildOnly'
 740     }
 741
 742     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 743
 744     if (validOsPrTriggerArchConfigs == null) {
 745         return false
 746     }
 747
 748     if (validOsPrTriggerArchConfigs[architecture] != null) {
 749         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 750
 751         if (!(configuration in validOsPrTriggerConfigs)) {
 752             return false
 753         }
 754     } else {
 755         return false
 756     }
 757
 758     return true
 759 }
 760
 761 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 762     // 2 hours (120 minutes) is the default timeout
 763     def timeout = 120
 764     def innerLoop = (scenario == "innerloop")
 765
 766     if (!innerLoop) {
 767         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 768         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 769         timeout = 240
 770     }
 771
 772     if (!isBuildOnly) {
 773         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 774         if (isGCStressRelatedTesting(scenario)) {
 775             timeout = 4320
 776         }
 777         else if (isCoreFxScenario(scenario)) {
 778             timeout = 360
 779         }
 780         else if (isJitStressScenario(scenario)) {
 781             timeout = 300
 782         }
 783         else if (isR2RBaselineScenario(scenario)) {
 784             timeout = 240
 785         }
 786         else if (isLongGc(scenario)) {
 787             timeout = 1440
 788         }
 789         else if (isJitDiff(scenario)) {
 790             timeout = 240
 791         }
 792         else if (isGcReliabilityFramework(scenario)) {
 793             timeout = 1440
 794         }
 795         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 796             timeout = 240
 797         }
 798
 799         if (architecture == 'arm') {
 800             // ARM32 machines are particularly slow.
 801             timeout += 120
 802         }
 803     }
 804
 805     if (configuration == 'Debug') {
 806         // Debug runs can be very slow. Add an hour.
 807         timeout += 60
 808     }
 809
 810     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 811         // AltJit runs compile all methods twice.
 812         timeout *= 2
 813     }
 814
 815     // If we've changed the timeout from the default, set it in the job.
 816
 817     if (timeout != 120) {
 818         Utilities.setJobTimeout(newJob, timeout)
 819     }
 820 }
 821
 822 def static getJobFolder(def scenario) {
 823     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 824         return 'jitstress'
 825     }
 826     if (scenario == 'illink') {
 827         return 'illink'
 828     }
 829     return ''
 830 }
 831
 832 def static getStressModeDisplayName(def scenario) {
 833     def displayStr = ''
 834     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 835         def prefixLength = 'COMPlus_'.length()
 836         if (k.length() >= prefixLength) {
 837             def modeName = k.substring(prefixLength, k.length())
 838             displayStr += ' ' + modeName + '=' + v
 839         }
 840     }
 841
 842     if (isCoreFxScenario(scenario)) {
 843         displayStr = ('CoreFx ' + displayStr).trim()
 844     }
 845
 846     return displayStr
 847 }
 848
 849 def static getR2RDisplayName(def scenario) {
 850     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 851     def displayStr = scenario
 852     def prefixLength = 'r2r_'.length()
 853     if (displayStr.length() >= prefixLength) {
 854         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 855     } else if (scenario == 'r2r') {
 856         displayStr = "R2R"
 857     }
 858     return displayStr
 859 }
 860
 861 //
 862 // Functions to create an environment script.
 863 //      envScriptCreate -- initialize the script (call first)
 864 //      envScriptFinalize -- finalize the script (call last)
 865 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 866 //      envScriptAppendExistingScript -- append an existing script to the generated script
 867 //
 868 // Each script returns a string of commands. Concatenate all the strings together before
 869 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 870 //
 871
 872 // Initialize the environment setting script.
 873 def static envScriptCreate(def os, def stepScriptLocation) {
 874     def stepScript = ''
 875     if (os == 'Windows_NT') {
 876         stepScript += "echo Creating TestEnv script\r\n"
 877         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 878
 879         // Create at least an empty script.
 880         stepScript += "echo. > ${stepScriptLocation}\r\n"
 881     }
 882     else {
 883         stepScript += "echo Creating environment setting script\n"
 884         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 885     }
 886
 887     return stepScript
 888 }
 889
 890 // Generates the string for setting stress mode variables.
 891 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 892     def stepScript = ''
 893     if (os == 'Windows_NT') {
 894         stressModeVars.each{ k, v ->
 895             // Write out what we are writing to the script file
 896             stepScript += "echo Setting ${k}=${v}\r\n"
 897             // Write out the set itself to the script file`
 898             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 899         }
 900     }
 901     else {
 902         stressModeVars.each{ k, v ->
 903             // Write out what we are writing to the script file
 904             stepScript += "echo Setting ${k}=${v}\n"
 905             // Write out the set itself to the script file`
 906             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 907         }
 908     }
 909
 910     return stepScript
 911 }
 912
 913 // Append an existing script to an environment script.
 914 // Returns string of commands to do this.
 915 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 916     assert (os == 'Windows_NT')
 917     def stepScript = ''
 918
 919     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 920     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 921
 922     return stepScript
 923 }
 924
 925 // Finalize an environment setting script.
 926 // Returns string of commands to do this.
 927 def static envScriptFinalize(def os, def stepScriptLocation) {
 928     def stepScript = ''
 929
 930     if (os == 'Windows_NT') {
 931         // Display the resulting script. This is useful when looking at the output log file.
 932         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 933         stepScript += "type ${stepScriptLocation}\r\n"
 934     }
 935     else {
 936         stepScript += "chmod +x ${stepScriptLocation}\n"
 937     }
 938
 939     return stepScript
 940 }
 941
 942 def static isNeedDocker(def architecture, def os, def isBuild) {
 943     if (isBuild) {
 944         if (architecture == 'x86' && os == 'Ubuntu') {
 945             return true
 946         }
 947         else if (architecture == 'armem') {
 948             return true
 949         }
 950         else if (architecture == 'arm') {
 951             if (os == 'Ubuntu') {
 952                 return true
 953             }
 954         }
 955     }
 956     else {
 957         if (architecture == 'x86' && os == 'Ubuntu') {
 958             return true
 959         }
 960     }
 961     return false
 962 }
 963
 964 def static getDockerImageName(def architecture, def os, def isBuild) {
 965     // We must change some docker private images to official later
 966     if (isBuild) {
 967         if (architecture == 'x86' && os == 'Ubuntu') {
 968             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 969         }
 970         else if (architecture == 'armem') {
 971             if (os == 'Ubuntu') {
 972                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 973             }
 974             else if (os == 'Ubuntu16.04') {
 975                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 976             }
 977             else if (os == 'Tizen') {
 978                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 979             }
 980         }
 981         else if (architecture == 'arm') {
 982             if (os == 'Ubuntu') {
 983                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20170319080304"
 984             }
 985         }
 986     }
 987     else {
 988         if (architecture == 'x86' && os == 'Ubuntu') {
 989             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 990         }
 991     }
 992     println("Unknown architecture to use docker: ${architecture} ${os}");
 993     assert false
 994 }
 995
 996
 997 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
 998 def static jobRequiresLimitedHardware(def architecture, def os) {
 999     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
1000         // These test jobs require ARM64 hardware
1001         return true
1002     }
1003     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1004         // These test jobs require Linux/arm32 hardware
1005         return true
1006     }
1007     else {
1008         return false
1009     }
1010 }
1011
1012 // Calculates the name of the build job based on some typical parameters.
1013 //
1014 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1015     // If the architecture is x64, do not add that info into the build name.
1016     // Need to change around some systems and other builds to pick up the right builds
1017     // to do that.
1018
1019     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1020     if (isBuildOnly) {
1021         suffix += '_bld'
1022     }
1023     def baseName = ''
1024     switch (architecture) {
1025         case 'x64':
1026             if (scenario == 'normal') {
1027                 // For now we leave x64 off of the name for compatibility with other jobs
1028                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1029             }
1030             else if (scenario == 'formatting') {
1031                 // we don't care about the configuration for the formatting job. It runs all configs
1032                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1033             }
1034             else {
1035                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1036             }
1037             break
1038         case 'arm64':
1039             if (os.toLowerCase() == "windows_nt") {
1040                 // These are cross builds
1041                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1042             }
1043             else {
1044                 // Defaults to a small page size set of machines.
1045                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1046             }
1047             break
1048         case 'armem':
1049             // These are cross builds
1050             if (os == 'Tizen') {
1051                 // ABI: softfp
1052                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1053             }
1054             else {
1055                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1056             }
1057             break
1058         case 'armlb':
1059         case 'arm':
1060             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1061             break
1062         case 'x86':
1063         case 'x86_arm_altjit':
1064         case 'x64_arm64_altjit':
1065             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1066             break
1067         default:
1068             println("Unknown architecture: ${architecture}");
1069             assert false
1070             break
1071     }
1072
1073     return baseName + suffix
1074 }
1075
1076 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1077
1078     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1079     if (os == 'Windows_NT') {
1080         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1081             if (!(branch in Constants.WindowsArm64Branches)) {
1082                 return
1083             }
1084         }
1085     }
1086
1087     // Check scenario.
1088     switch (scenario) {
1089         case 'innerloop':
1090             break
1091         case 'normal':
1092             switch (architecture) {
1093                 case 'x64':
1094                 case 'x86':
1095                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1096                         addPeriodicTriggerHelper(job, '@daily')
1097                     }
1098                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1099                         addGithubPushTriggerHelper(job)
1100                     }
1101                     break
1102                 case 'arm':
1103                     if (os == 'Windows_NT') {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     else {
1107                         // Currently no push triggers, with limited arm Linux hardware.
1108                         // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1109                         assert os == 'Ubuntu'
1110                         if (isFlowJob) {
1111                             addPeriodicTriggerHelper(job, '@daily')
1112                         }
1113                     }
1114                     break
1115                 case 'armem':
1116                 case 'armlb':
1117                 case 'x86_arm_altjit':
1118                 case 'x64_arm64_altjit':
1119                     addGithubPushTriggerHelper(job)
1120                     break
1121                 case 'arm64':
1122                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1123                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1124                     break
1125                 default:
1126                     println("Unknown architecture: ${architecture}");
1127                     assert false
1128                     break
1129             }
1130             break
1131         case 'r2r':
1132             assert !(os in bidailyCrossList)
1133             // r2r gets a push trigger for checked/release
1134             if (configuration == 'Checked' || configuration == 'Release') {
1135                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1136                 if (architecture == 'x64' && os != 'OSX10.12') {
1137                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1138                     if (isFlowJob || os == 'Windows_NT') {
1139                         addGithubPushTriggerHelper(job)
1140                     }
1141                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1142                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1143                     if (isFlowJob) {
1144                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1145                     }
1146                 }
1147                 // For x86, only add per-commit jobs for Windows
1148                 else if (architecture == 'x86') {
1149                     if (os == 'Windows_NT') {
1150                         addGithubPushTriggerHelper(job)
1151                     }
1152                 }
1153                 // arm64 r2r jobs should only run daily.
1154                 else if (architecture == 'arm64') {
1155                     if (os == 'Windows_NT') {
1156                         addPeriodicTriggerHelper(job, '@daily')
1157                     }
1158                 }
1159             }
1160             break
1161         case 'r2r_jitstress1':
1162         case 'r2r_jitstress2':
1163         case 'r2r_jitstressregs1':
1164         case 'r2r_jitstressregs2':
1165         case 'r2r_jitstressregs3':
1166         case 'r2r_jitstressregs4':
1167         case 'r2r_jitstressregs8':
1168         case 'r2r_jitstressregs0x10':
1169         case 'r2r_jitstressregs0x80':
1170         case 'r2r_jitstressregs0x1000':
1171         case 'r2r_jitminopts':
1172         case 'r2r_jitforcerelocs':
1173         case 'r2r_gcstress15':
1174             assert !(os in bidailyCrossList)
1175
1176             // GCStress=C is currently not supported on OS X
1177             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1178                 break
1179             }
1180
1181             // GC Stress 15 r2r gets a push trigger for checked/release
1182             if (configuration == 'Checked' || configuration == 'Release') {
1183                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1184                 if (architecture == 'x64') {
1185                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1186                     if (isFlowJob || os == 'Windows_NT') {
1187                         // Add a weekly periodic trigger
1188                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1189                     }
1190                 }
1191                 // For x86, only add per-commit jobs for Windows
1192                 else if (architecture == 'x86') {
1193                     if (os == 'Windows_NT') {
1194                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1195                     }
1196                 }
1197             }
1198             break
1199         case 'longgc':
1200             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1201             assert configuration == 'Release'
1202             assert architecture == 'x64'
1203             addPeriodicTriggerHelper(job, '@daily')
1204             // TODO: Add once external email sending is available again
1205             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1206             break
1207         case 'gcsimulator':
1208             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1209             assert configuration == 'Release'
1210             assert architecture == 'x64'
1211             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1212             // TODO: Add once external email sending is available again
1213             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1214             break
1215         case 'standalone_gc':
1216             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1217             assert (configuration == 'Release' || configuration == 'Checked')
1218             // TODO: Add once external email sending is available again
1219             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1220             addPeriodicTriggerHelper(job, '@daily')
1221             break
1222         case 'gc_reliability_framework':
1223             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1224             assert (configuration == 'Release' || configuration == 'Checked')
1225             // Only triggered by phrase.
1226             break
1227         case 'ilrt':
1228             assert !(os in bidailyCrossList)
1229             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1230             if (architecture == 'x64' && configuration == 'Release') {
1231                 // We don't expect to see a job generated except in these scenarios
1232                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1233                 if (isFlowJob || os == 'Windows_NT') {
1234                     addPeriodicTriggerHelper(job, '@daily')
1235                 }
1236             }
1237             break
1238         case 'jitdiff':
1239             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1240             assert configuration == 'Checked'
1241             assert (architecture == 'x64' || architecture == 'x86')
1242             addGithubPushTriggerHelper(job)
1243             break
1244         case 'formatting':
1245             assert (os == 'Windows_NT' || os == "Ubuntu")
1246             assert architecture == 'x64'
1247             addGithubPushTriggerHelper(job)
1248             break
1249         case 'jitstressregs1':
1250         case 'jitstressregs2':
1251         case 'jitstressregs3':
1252         case 'jitstressregs4':
1253         case 'jitstressregs8':
1254         case 'jitstressregs0x10':
1255         case 'jitstressregs0x80':
1256         case 'jitstressregs0x1000':
1257         case 'minopts':
1258         case 'forcerelocs':
1259         case 'jitstress1':
1260         case 'jitstress2':
1261         case 'jitstress2_jitstressregs1':
1262         case 'jitstress2_jitstressregs2':
1263         case 'jitstress2_jitstressregs3':
1264         case 'jitstress2_jitstressregs4':
1265         case 'jitstress2_jitstressregs8':
1266         case 'jitstress2_jitstressregs0x10':
1267         case 'jitstress2_jitstressregs0x80':
1268         case 'jitstress2_jitstressregs0x1000':
1269         case 'tailcallstress':
1270         case 'jitsse2only':
1271         case 'jitnosimd':
1272         case 'jitnox86hwintrinsic':
1273         case 'jitincompletehwintrinsic':
1274         case 'jitx86hwintrinsicnoavx':
1275         case 'jitx86hwintrinsicnoavx2':
1276         case 'jitx86hwintrinsicnosimd':
1277         case 'corefx_baseline':
1278         case 'corefx_minopts':
1279         case 'corefx_jitstress1':
1280         case 'corefx_jitstress2':
1281         case 'corefx_jitstressregs1':
1282         case 'corefx_jitstressregs2':
1283         case 'corefx_jitstressregs3':
1284         case 'corefx_jitstressregs4':
1285         case 'corefx_jitstressregs8':
1286         case 'corefx_jitstressregs0x10':
1287         case 'corefx_jitstressregs0x80':
1288         case 'corefx_jitstressregs0x1000':
1289         case 'zapdisable':
1290             if (os == 'CentOS7.1') {
1291                 break
1292             }
1293             if (os in bidailyCrossList) {
1294                 break
1295             }
1296             assert (os == 'Windows_NT') || (os in Constants.crossList)
1297             if (jobRequiresLimitedHardware(architecture, os)) {
1298                 addPeriodicTriggerHelper(job, '@weekly')
1299             }
1300             else {
1301                 addPeriodicTriggerHelper(job, '@daily')
1302             }
1303             break
1304         case 'heapverify1':
1305         case 'gcstress0x3':
1306             if (os == 'CentOS7.1') {
1307                 break
1308             }
1309             if (os in bidailyCrossList) {
1310                 break
1311             }
1312             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1313                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1314                 break
1315             }
1316             assert (os == 'Windows_NT') || (os in Constants.crossList)
1317             addPeriodicTriggerHelper(job, '@weekly')
1318             break
1319         case 'gcstress0xc':
1320         case 'gcstress0xc_zapdisable':
1321         case 'gcstress0xc_zapdisable_jitstress2':
1322         case 'gcstress0xc_zapdisable_heapverify1':
1323         case 'gcstress0xc_jitstress1':
1324         case 'gcstress0xc_jitstress2':
1325         case 'gcstress0xc_minopts_heapverify1':
1326             if (os == 'CentOS7.1') {
1327                 break
1328             }
1329             if (os == 'OSX10.12') {
1330                 // GCStress=C is currently not supported on OS X
1331                 break
1332             }
1333             if (os in bidailyCrossList) {
1334                 break
1335             }
1336             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1337                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1338                 break
1339             }
1340             assert (os == 'Windows_NT') || (os in Constants.crossList)
1341             addPeriodicTriggerHelper(job, '@weekly')
1342             break
1343
1344         case 'illink':
1345             // Testing on other operating systems TBD
1346             assert (os == 'Windows_NT' || os == 'Ubuntu')
1347             if (architecture == 'x64' || architecture == 'x86') {
1348                 if (configuration == 'Checked') {
1349                     addPeriodicTriggerHelper(job, '@daily')
1350                 }
1351             }
1352             break
1353
1354         case 'tieredcompilation':
1355         case 'corefx_tieredcompilation':
1356             // No periodic jobs just yet, still testing
1357             break
1358
1359         default:
1360             println("Unknown scenario: ${scenario}");
1361             assert false
1362             break
1363     }
1364     return
1365 }
1366
1367 // **************************
1368 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1369 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1370 // builds will do a couple extra steps.
1371 // **************************
1372
1373 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1374 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1375 // it's just used for internal builds
1376 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1377 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1378     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1379
1380     if (isWindowsBuildOnlyJob) {
1381         return
1382     }
1383
1384     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1385     // Non pull request builds.
1386     if (!isPR) {
1387         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1388         return
1389     }
1390
1391      def arm64Users = [
1392         'AndyAyersMS',
1393         'briansull',
1394         'BruceForstall',
1395         'CarolEidt',
1396         'cmckinsey',
1397         'erozenfeld',
1398         'janvorli',
1399         'jashook',
1400         'JosephTremoulet',
1401         'pgodeq',
1402         'pgavlin',
1403         'rartemev',
1404         'russellhadley',
1405         'RussKeldorph',
1406         'sandreenko',
1407         'sdmaclea',
1408         'swaroop-sridhar',
1409         'jkotas',
1410         'markwilkie',
1411         'weshaggard'
1412     ]
1413
1414     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1415     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1416     def osGroup = getOSGroup(os)
1417     switch (architecture) {
1418         case 'x64': // editor brace matching: {
1419             if (scenario == 'formatting') {
1420                 assert configuration == 'Checked'
1421                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1422                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1423                 }
1424
1425                 break
1426             }
1427
1428             switch (os) {
1429                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1430                 case 'Debian8.4':
1431                 case 'RHEL7.2':
1432                     if (scenario == 'innerloop') {
1433                         assert !isFlowJob
1434                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1435                     }
1436                     else if (scenario == 'normal') {
1437                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1438                     }
1439                     break
1440
1441                 case 'Ubuntu16.04':
1442                     assert !isFlowJob
1443                     assert scenario != 'innerloop'
1444                     // Distinguish with the other architectures (arm and x86)
1445                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1446                     break
1447
1448                 case 'Fedora24':
1449                 case 'Ubuntu16.10':
1450                     assert !isFlowJob
1451                     assert scenario != 'innerloop'
1452                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1453                     break
1454
1455                 case 'Ubuntu':
1456                     if (scenario == 'illink') {
1457                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1458                         break
1459                     }
1460                     // fall through
1461
1462                 case 'OSX10.12':
1463                     // Triggers on the non-flow jobs aren't necessary here
1464                     // Corefx testing uses non-flow jobs.
1465                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1466                         break
1467                     }
1468                     switch (scenario) {
1469                         case 'innerloop':
1470                             // PR Triggered jobs. These jobs will run pri0 tests.
1471                             if (configuration == 'Checked') {
1472                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1473                             }
1474                             break
1475
1476                         case 'normal':
1477                             // OSX uses checked for default PR tests
1478                             if (configuration == 'Checked') {
1479                                 // Default trigger
1480                                 assert !job.name.contains("centos")
1481                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1482                             }
1483                             break
1484
1485                         case 'jitdiff':
1486                             if (configuration == 'Checked') {
1487                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1488                             }
1489                             break
1490
1491                         case 'ilrt':
1492                             if (configuration == 'Release') {
1493                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1494                             }
1495                             break
1496
1497                         case 'longgc':
1498                             if (configuration == 'Release') {
1499                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1500                             }
1501                             break
1502
1503                         case 'gcsimulator':
1504                             if (configuration == 'Release') {
1505                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1506                             }
1507                             break
1508
1509                         case 'standalone_gc':
1510                             if (configuration == 'Release' || configuration == 'Checked') {
1511                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1512                             }
1513                             break
1514
1515                         case 'gc_reliability_framework':
1516                             if (configuration == 'Release' || configuration == 'Checked') {
1517                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1518                             }
1519                             break
1520
1521                         default:
1522                             if (isJitStressScenario(scenario)) {
1523                                 def displayStr = getStressModeDisplayName(scenario)
1524                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1525                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1526                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1527                             }
1528                             else if (isR2RScenario(scenario)) {
1529                                 if (configuration == 'Release' || configuration == 'Checked') {
1530                                     def displayStr = getR2RDisplayName(scenario)
1531                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1532                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1533                                 }
1534                             }
1535                             else {
1536                                 println("Unknown scenario: ${scenario}");
1537                                 assert false
1538                             }
1539                             break
1540
1541                     }
1542                     break
1543
1544                 case 'CentOS7.1':
1545                     switch (scenario) {
1546                         case 'innerloop':
1547                             // CentOS uses checked for default PR tests while debug is build only
1548                             if (configuration == 'Debug') {
1549                                 // Default trigger
1550                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1551                             }
1552
1553                             // Make sure this is a flow job to get build and test.
1554                             if (configuration == 'Checked' && isFlowJob) {
1555                                 assert job.name.contains("flow")
1556                                 // Default trigger
1557                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1558                             }
1559                             break
1560
1561                         case 'normal':
1562                             // Make sure this is a flow job to get build and test.
1563                             if (configuration == 'Checked' && isFlowJob) {
1564                                 assert job.name.contains("flow")
1565                                 // Default trigger
1566                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1567                             }
1568                             break
1569
1570                         default:
1571                             if (isR2RScenario(scenario)) {
1572                                 if (configuration == 'Release' || configuration == 'Checked') {
1573                                     def displayStr = getR2RDisplayName(scenario)
1574                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1575                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1576                                 }
1577                             }
1578                             break
1579
1580                     }
1581                     break
1582
1583                 case 'Windows_NT':
1584                     switch (scenario) {
1585                         case 'innerloop':
1586                             // Default trigger
1587                             if (configuration == 'Checked' || configuration == 'Release') {
1588                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1589                             }
1590                             break
1591
1592                         case 'normal':
1593                             if (configuration == 'Checked') {
1594                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1595                             }
1596                             break
1597
1598                         case 'jitdiff':
1599                             if (configuration == 'Checked') {
1600                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1601                             }
1602                             break
1603
1604                         case 'ilrt':
1605                             if (configuration == 'Release') {
1606                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1607                             }
1608                             break
1609
1610                         case 'longgc':
1611                             if (configuration == 'Release') {
1612                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1613                             }
1614                             break
1615
1616                         case 'gcsimulator':
1617                             if (configuration == 'Release') {
1618                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1619                             }
1620                             break
1621
1622                         case 'standalone_gc':
1623                             if (configuration == 'Release' || configuration == 'Checked') {
1624                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1625                             }
1626                             break
1627
1628                         case 'gc_reliability_framework':
1629                             if (configuration == 'Release' || configuration == 'Checked') {
1630                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1631                             }
1632                             break
1633
1634                         case 'illink':
1635                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1636                             break
1637
1638                         default:
1639                             if (isJitStressScenario(scenario)) {
1640                                 def displayStr = getStressModeDisplayName(scenario)
1641                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1642                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1643                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1644                             }
1645                             else if (isR2RScenario(scenario)) {
1646                                 if (configuration == 'Release' || configuration == 'Checked') {
1647                                     def displayStr = getR2RDisplayName(scenario)
1648                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1649                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1650                                 }
1651                             }
1652                             else {
1653                                 println("Unknown scenario: ${scenario}");
1654                                 assert false
1655                             }
1656                             break
1657
1658                     }
1659                     break
1660
1661                 default:
1662                     println("Unknown os: ${os}");
1663                     assert false
1664                     break
1665
1666             }
1667
1668             break
1669         // editor brace matching: }
1670
1671         case 'armem': // editor brace matching: {
1672             job.with {
1673                 publishers {
1674                     azureVMAgentPostBuildAction {
1675                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1676                     }
1677                 }
1678             }
1679
1680             switch (os) {
1681                 case 'Ubuntu':
1682                 case 'Ubuntu16.04':
1683                     assert scenario != 'innerloop'
1684                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1685                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1686                     break
1687
1688                 case 'Tizen':
1689                     architecture = 'armel'
1690
1691                     assert scenario != 'innerloop'
1692                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1693                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1694                     break
1695             }
1696
1697             break
1698         // editor brace matching: }
1699
1700         case 'armlb':
1701         case 'arm': // editor brace matching: {
1702
1703             // Triggers on the non-flow jobs aren't necessary
1704             if (!isFlowJob) {
1705                 break
1706             }
1707
1708             // Set up a private trigger
1709             def contextString = "${os} ${architecture} Cross ${configuration}"
1710             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1711             if (scenario == 'innerloop') {
1712                 contextString += " Innerloop"
1713                 triggerString += "\\W+Innerloop"
1714             }
1715             else {
1716                 contextString += " ${scenario}"
1717                 triggerString += "\\W+${scenario}"
1718             }
1719
1720             if (configuration == 'Debug') {
1721                 contextString += " Build"
1722                 triggerString += "\\W+Build"
1723             } else {
1724                 contextString += " Build and Test"
1725                 triggerString += "\\W+Build and Test"
1726             }
1727
1728             triggerString += ".*"
1729
1730             switch (os) {
1731                 case 'Ubuntu':
1732                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1733                         break
1734                     }
1735
1736                     if (scenario == 'innerloop') {
1737                         if (configuration == 'Checked') {
1738                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1739                         }
1740                     }
1741                     else {
1742                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1743                     }
1744                     break
1745
1746                 case 'Windows_NT':
1747                     if (architecture == "armlb") {
1748                         // Disable armlb windows jobs
1749                         break
1750                     }
1751                     switch (scenario) {
1752                         case 'innerloop':
1753                             // Only Checked is an innerloop trigger.
1754                             if (configuration == 'Checked')
1755                             {
1756                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1757                             }
1758                             break
1759                         case 'normal':
1760                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1761                             break
1762                         default:
1763                             // Stress jobs will use this code path.
1764                             if (isArmWindowsScenario(scenario)) {
1765                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1766                             }
1767                             break
1768                     }
1769                     break
1770                 default:
1771                     println("NYI os: ${os}");
1772                     assert false
1773                     break
1774             }
1775             break
1776         // editor brace matching: }
1777         case 'arm64': // editor brace matching: {
1778             // Set up a private trigger
1779             def contextString = "${os} ${architecture} Cross ${configuration}"
1780             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1781
1782             if (scenario == 'innerloop') {
1783                 contextString += " Innerloop"
1784                 triggerString += "\\W+Innerloop"
1785             }
1786             else {
1787                 contextString += " ${scenario}"
1788                 triggerString += "\\W+${scenario}"
1789             }
1790
1791             if (configuration == 'Debug') {
1792                 contextString += " Build"
1793                 triggerString += "\\W+Build"
1794             } else {
1795                 contextString += " Build and Test"
1796                 triggerString += "\\W+Build and Test"
1797             }
1798
1799             triggerString += ".*"
1800
1801             switch (os) {
1802                 case 'Ubuntu':
1803                 case 'Ubuntu16.04':
1804                     switch (scenario) {
1805                         case 'innerloop':
1806                             if (configuration == 'Debug' && !isFlowJob) {
1807                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1808                             }
1809
1810                             break
1811                         case 'normal':
1812                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1813                             break
1814                         default:
1815                             if (isR2RScenario(scenario)) {
1816                                 if (configuration == 'Checked' || configuration == 'Release') {
1817                                     def displayStr = getR2RDisplayName(scenario)
1818                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1819                                 }
1820                             }
1821                             break
1822                     }
1823                     break
1824
1825                 case 'Windows_NT':
1826                     // Triggers on the non-flow jobs aren't necessary here
1827                     if (!isFlowJob) {
1828                         break
1829                     }
1830
1831                     assert isArmWindowsScenario(scenario)
1832                     switch (scenario) {
1833                         case 'innerloop':
1834                             if (configuration == 'Checked') {
1835                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1836                             }
1837
1838                             break
1839                         case 'normal':
1840                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1841                             break
1842                         default:
1843                             // Stress jobs will use this code path.
1844                             if (isArmWindowsScenario(scenario)) {
1845                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1846                             }
1847                             break
1848                     }
1849                     break
1850                 default:
1851                     println("NYI os: ${os}");
1852                     assert false
1853                     break
1854             }
1855             break
1856
1857         // editor brace matching: }
1858         case 'x86': // editor brace matching: {
1859             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1860             if (os == 'Ubuntu') {
1861                 // Triggers on the non-flow jobs aren't necessary here
1862                 if (!isFlowJob) {
1863                     break
1864                 }
1865
1866                 // on-demand only for ubuntu x86
1867                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1868                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1869                 break
1870
1871             }
1872             switch (scenario) {
1873                 case 'innerloop':
1874                     if (configuration == 'Checked' || configuration == 'Release') {
1875                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1876                     }
1877                     break
1878
1879                 case 'normal':
1880                     if (configuration == 'Checked') {
1881                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1882                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1883                     }
1884                     break
1885
1886                 case 'ilrt':
1887                     if (configuration == 'Release') {
1888                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1889                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1890                     }
1891                     break
1892
1893                 case 'longgc':
1894                     if (configuration == 'Release') {
1895                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1896                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1897                     }
1898                     break
1899
1900                 case 'gcsimulator':
1901                     if (configuration == 'Release') {
1902                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1903                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1904                     }
1905                     break
1906
1907                 case 'standalone_gc':
1908                     if (configuration == 'Release' || configuration == 'Checked') {
1909                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1910                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1911                     }
1912                     break
1913
1914                 case 'illink':
1915                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1916                     break
1917
1918                 default:
1919                     if (isJitStressScenario(scenario)) {
1920                         def displayStr = getStressModeDisplayName(scenario)
1921                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1922                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1923                     }
1924                     else if (isR2RScenario(scenario)) {
1925                         if (configuration == 'Release' || configuration == 'Checked') {
1926                             def displayStr = getR2RDisplayName(scenario)
1927                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1928                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1929                         }
1930                     }
1931                     else {
1932                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1933                         assert false
1934                     }
1935                     break
1936
1937             }
1938             break
1939
1940          // editor brace matching: }
1941         case 'x64_arm64_altjit':
1942         case 'x86_arm_altjit': // editor brace matching: {
1943             assert (os == 'Windows_NT')
1944             switch (scenario) {
1945                 case 'normal':
1946                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1947                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1948                     break
1949                 default:
1950                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1951                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1952                     break
1953             }
1954             break
1955
1956         // editor brace matching: }
1957         default:
1958             println("Unknown architecture: ${architecture}");
1959             assert false
1960             break
1961     }
1962 }
1963
1964 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1965     def buildCommands = []
1966     def osGroup = getOSGroup(os)
1967     def lowerConfiguration = configuration.toLowerCase()
1968
1969     def priority = '1'
1970     if (scenario == 'innerloop') {
1971         priority = '0'
1972     }
1973
1974     def doCoreFxTesting = isCoreFxScenario(scenario)
1975
1976     // Calculate the build steps, archival, and xunit results
1977     switch (os) {
1978         case 'Windows_NT': // editor brace matching: {
1979             switch (architecture) {
1980                 case 'x64':
1981                 case 'x86':
1982                 case 'x86_arm_altjit':
1983                 case 'x64_arm64_altjit':
1984                     def arch = architecture
1985                     def buildOpts = ''
1986                     if (architecture == 'x86_arm_altjit') {
1987                         arch = 'x86'
1988                     }
1989                     else if (architecture == 'x64_arm64_altjit') {
1990                         arch = 'x64'
1991                     }
1992
1993                     if (scenario == 'formatting') {
1994                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1995                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1996                         break
1997                     }
1998
1999                     if (scenario == 'illink') {
2000                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2001                     }
2002
2003                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2004                     if ((lowerConfiguration == 'release') &&
2005                         (scenario in Constants.basicScenarios) &&
2006                         (architecture != 'x86_arm_altjit') &&
2007                         (architecture != 'x64_arm64_altjit')) {
2008
2009                         buildOpts += ' -enforcepgo'
2010                     }
2011
2012                     if (doCoreFxTesting) {
2013                         buildOpts += ' skiptests';
2014                     } else {
2015                         buildOpts += " -priority=${priority}"
2016                     }
2017
2018                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2019                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2020                     // 35 characters long.
2021
2022                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2023
2024                     if (!isBuildOnly) {
2025                         def runtestArguments = ''
2026                         def testOpts = 'collectdumps'
2027
2028                         if (isR2RScenario(scenario)) {
2029
2030                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2031                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2032                             // to cause the tests to be crossgen'ed.
2033
2034                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2035                                 testOpts += ' crossgenaltjit protononjit.dll'
2036                             } else {
2037                                 testOpts += ' crossgen'
2038                             }
2039
2040                             testOpts += ' runcrossgentests'
2041                         }
2042                         else if (scenario == 'jitdiff') {
2043                             testOpts += ' jitdisasm crossgen'
2044                         }
2045                         else if (scenario == 'ilrt') {
2046                             testOpts += ' ilasmroundtrip'
2047                         }
2048                         else if (isLongGc(scenario)) {
2049                             testOpts += " ${scenario} sequential"
2050                         }
2051                         else if (scenario == 'standalone_gc') {
2052                             testOpts += ' gcname clrgc.dll'
2053                         }
2054                         else if (scenario == 'illink') {
2055                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2056                         }
2057
2058                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2059                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2060                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2061                         // which is read by the xunit harness.)
2062                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2063                         {
2064                             def timeout = 1800000
2065                             testOpts += " timeout ${timeout}"
2066                         }
2067
2068                         // If we are running a stress mode, we should write out the set of key
2069                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2070
2071                         def envScriptPath = ''
2072                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2073                             def buildCommandsStr = ''
2074                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2075                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2076
2077                             if (isJitStressScenario(scenario)) {
2078                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2079                             }
2080                             else if (isR2RStressScenario(scenario)) {
2081                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2082                             }
2083
2084                             if (architecture == 'x86_arm_altjit') {
2085                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2086                             }
2087                             else if (architecture == 'x64_arm64_altjit') {
2088                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2089                             }
2090
2091                             envScriptFinalize(os, envScriptPath)
2092
2093                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2094                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2095                             // into a single script.
2096                             buildCommands += buildCommandsStr
2097                         }
2098                         else if (architecture == 'x86_arm_altjit') {
2099                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2100                         }
2101                         else if (architecture == 'x64_arm64_altjit') {
2102                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2103                         }
2104                         if (envScriptPath != '') {
2105                             testOpts += " TestEnv ${envScriptPath}"
2106                         }
2107
2108                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2109
2110                         if (doCoreFxTesting) {
2111                             def workspaceRelativeFxRoot = "_/fx"
2112                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2113
2114                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2115
2116                             // Archive and process (only) the test results
2117                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2118                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2119
2120                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2121                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2122                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2123                         }
2124                         else if (isGcReliabilityFramework(scenario)) {
2125                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2126                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2127                         }
2128                         else {
2129                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2130                         }
2131                     } // end if (!isBuildOnly)
2132
2133                     if (!doCoreFxTesting) {
2134                         // Run the rest of the build
2135                         // Build the mscorlib for the other OS's
2136                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2137                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2138
2139                         if (arch == 'x64') {
2140                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2141                         }
2142
2143                         // Zip up the tests directory so that we don't use so much space/time copying
2144                         // 10s of thousands of files around.
2145                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2146
2147                         if (!isJitStressScenario(scenario)) {
2148                             // For Windows, pull full test results and test drops for x86/x64.
2149                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2150                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2151                         }
2152
2153                         if (scenario == 'jitdiff') {
2154                             // retrieve jit-dasm output for base commit, and run jit-diff
2155                             if (!isBuildOnly) {
2156                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2157                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2158                             }
2159                         }
2160
2161                         if (!isBuildOnly) {
2162                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2163                         }
2164                     }
2165                     break
2166                 case 'armlb':
2167                 case 'arm':
2168                     assert isArmWindowsScenario(scenario)
2169
2170                     def buildArchitecture = 'arm'
2171
2172                     def buildOpts = ''
2173
2174                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2175                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2176                     if (architecture == 'armlb') {
2177                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2178                     }
2179
2180                     if (doCoreFxTesting) {
2181                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2182                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2183                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2184                         //
2185                         // buildOpts += ' skiptests';
2186                         buildOpts += " -priority=0"
2187                     } else {
2188                         buildOpts += " -priority=${priority}"
2189                     }
2190
2191                     // This is now a build only job. Do not run tests. Use the flow job.
2192                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2193
2194                     if (doCoreFxTesting) {
2195                         assert isBuildOnly
2196                         assert architecture == 'arm'
2197
2198                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2199                         // to determine the correct matching corefx version git commit hash.
2200                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2201
2202                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2203                         def envScriptPath = ''
2204                         def buildCommandsStr = ''
2205                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2206                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2207                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2208                         envScriptFinalize(os, envScriptPath)
2209                         buildCommands += buildCommandsStr
2210
2211                         def workspaceRelativeFxRootLinux = "_/fx"
2212                         def workspaceRelativeFxRootWin = "_\\fx"
2213                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2214
2215                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2216
2217                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2218                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2219                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2220
2221                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2222                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2223                     } else {
2224                         // Zip up the tests directory so that we don't use so much space/time copying
2225                         // 10s of thousands of files around.
2226                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2227
2228                         // Add archival.
2229                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2230                     }
2231                     break
2232                 case 'arm64':
2233                     assert isArmWindowsScenario(scenario)
2234
2235                     // This is now a build only job. Do not run tests. Use the flow job.
2236                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2237
2238                     // Zip up the tests directory so that we don't use so much space/time copying
2239                     // 10s of thousands of files around.
2240                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2241
2242                     // Add archival.
2243                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2244                     break
2245                 default:
2246                     println("Unknown architecture: ${architecture}");
2247                     assert false
2248                     break
2249             }
2250             break
2251         // end case 'Windows_NT'; editor brace matching: }
2252         case 'Ubuntu':
2253         case 'Ubuntu16.04':
2254         case 'Ubuntu16.10':
2255         case 'Debian8.4':
2256         case 'OSX10.12':
2257         case 'CentOS7.1':
2258         case 'RHEL7.2':
2259         case 'Tizen':
2260         case 'Fedora24': // editor brace matching: {
2261             switch (architecture) {
2262                 case 'x64':
2263                 case 'x86':
2264                     if (architecture == 'x86' && os == 'Ubuntu') {
2265                         // build and PAL test
2266                         def dockerImage = getDockerImageName(architecture, os, true)
2267                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2268                         dockerImage = getDockerImageName(architecture, os, false)
2269                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2270                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2271                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2272                         break
2273                     }
2274
2275                     if (scenario == 'formatting') {
2276                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2277                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2278                         break
2279                     }
2280
2281                     if (scenario == 'illink') {
2282                         assert(os == 'Ubuntu')
2283                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2284                     }
2285
2286                     if (!doCoreFxTesting) {
2287                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2288                         // only on supported OS platforms.
2289                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2290                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2291
2292                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2293                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2294
2295                         // Basic archiving of the build
2296                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2297                         // And pal tests
2298                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2299                     }
2300                     else {
2301                         // Corefx stress testing
2302                         assert os == 'Ubuntu'
2303                         assert architecture == 'x64'
2304                         assert lowerConfiguration == 'checked'
2305                         assert isJitStressScenario(scenario)
2306
2307                         // Build coreclr
2308                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2309
2310                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2311
2312                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2313                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2314                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2315                         buildCommands += envScriptCmds
2316
2317                         // Build and text corefx
2318                         def workspaceRelativeFxRoot = "_/fx"
2319                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2320
2321                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2322
2323                         // Archive and process (only) the test results
2324                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2325                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2326                     }
2327                     break
2328                 case 'arm64':
2329                     if (!doCoreFxTesting) {
2330                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2331
2332                         // HACK -- Arm64 does not have corefx jobs yet.
2333                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2334                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2335                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2336                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2337
2338                         // Basic archiving of the build
2339                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2340                     }
2341                     break
2342                 case 'armem':
2343                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2344                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2345
2346                     // default values for Ubuntu
2347                     def arm_abi = "arm"
2348                     def linuxCodeName = "trusty"
2349                     if (os == 'Ubuntu16.04') {
2350                         linuxCodeName = "xenial"
2351                     }
2352                     else if (os == 'Tizen') {
2353                         arm_abi = "armel"
2354                         linuxCodeName = "tizen"
2355                     }
2356
2357                     // Unzip the Windows test binaries first. Exit with 0
2358                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2359
2360                     // Unpack the corefx binaries
2361                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2362                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2363                     if (os != 'Tizen') {
2364                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2365                     }
2366                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2367                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2368                     if (os == 'Ubuntu') {
2369                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2370                     }
2371
2372                     // Call the ARM CI script to cross build and test using docker
2373                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2374                     --mode=docker \\
2375                     --${arm_abi} \\
2376                     --linuxCodeName=${linuxCodeName} \\
2377                     --buildConfig=${lowerConfiguration} \\
2378                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2379                     --coreFxBinDir=./bin/CoreFxBinDir \\
2380                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2381
2382                     // Basic archiving of the build, no pal tests
2383                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2384                     break
2385                 case 'arm':
2386                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2387                     assert (os == 'Ubuntu')
2388
2389                     // Add some useful information to the log file. Ignore return codes.
2390                     buildCommands += "uname -a || true"
2391
2392                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2393                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2394
2395                     def dockerImage = getDockerImageName(architecture, os, true)
2396                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm ${dockerImage} "
2397
2398                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross"
2399
2400                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2401                     // download the appropriate CoreFX packages.
2402                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2403                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2404
2405                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2406
2407                     // ZIP up for the test job (created in the flow job code):
2408                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2409                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2410                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2411                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2412
2413                     // These commands are assumed to be run from the root of the workspace.
2414                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2415                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2416
2417                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2418                     break
2419                 default:
2420                     println("Unknown architecture: ${architecture}");
2421                     assert false
2422                     break
2423             }
2424             break
2425         // editor brace matching: }
2426         default:
2427             println("Unknown os: ${os}");
2428             assert false
2429             break
2430     } // os
2431
2432     return buildCommands
2433 }
2434
2435 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2436 // Returns true if the job should be generated.
2437 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2438 {
2439     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2440     // All other scenarios do Pri-1 testing.
2441     if (scenario == 'innerloop' && !isPR) {
2442         return false
2443     }
2444
2445     // Tizen is only supported for armem architecture
2446     if (os == 'Tizen' && architecture != 'armem') {
2447         return false
2448     }
2449
2450     // Filter based on architecture.
2451
2452     switch (architecture) {
2453         case 'arm64':
2454         case 'arm':
2455             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2456                 return false
2457             }
2458             break
2459         case 'armem':
2460             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2461                 return false
2462             }
2463             break
2464         case 'armlb':
2465             // Do not create armlb jobs
2466             return false
2467         case 'x86_arm_altjit':
2468         case 'x64_arm64_altjit':
2469             if (os != 'Windows_NT') {
2470                 return false
2471             }
2472             break
2473         case 'x86':
2474             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2475                 return false
2476             }
2477             break
2478         case 'x64':
2479             // Everything implemented
2480             break
2481         default:
2482             println("Unknown architecture: ${architecture}")
2483             assert false
2484             break
2485     }
2486
2487     // Which (Windows) build only jobs are required?
2488
2489     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2490
2491     if (isBuildOnly) {
2492         switch (architecture) {
2493             case 'arm':
2494                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2495                 if (!isCoreFxScenario(scenario)) {
2496                     return false
2497                 }
2498                 break
2499             case 'x64':
2500             case 'x86':
2501                 if (!isNormalOrInnerloop) {
2502                     return false
2503                 }
2504                 break
2505             default:
2506                 return false
2507         }
2508     }
2509
2510     // Filter based on scenario.
2511
2512     if (isJitStressScenario(scenario)) {
2513         if (configuration != 'Checked') {
2514             return false
2515         }
2516
2517         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2518         if (!isEnabledOS) {
2519             return false
2520         }
2521
2522         switch (architecture) {
2523             case 'x64':
2524             case 'x86_arm_altjit':
2525             case 'x64_arm64_altjit':
2526                 break
2527
2528             case 'x86':
2529                 // x86 ubuntu: no stress modes
2530                 if (os == 'Ubuntu') {
2531                     return false
2532                 }
2533                 break
2534
2535             case 'arm':
2536                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2537                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2538                     return false
2539                 }
2540                 break
2541
2542             default:
2543                 // arm64, armlb: stress is handled through flow jobs.
2544                 // armem: no stress jobs for ARM emulator.
2545                 return false
2546         }
2547     }
2548     else if (isR2RScenario(scenario)) {
2549         if (os != 'Windows_NT') {
2550             return false
2551         }
2552         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2553         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2554             return false
2555         }
2556     }
2557     else {
2558         // Skip scenarios
2559         switch (scenario) {
2560             case 'ilrt':
2561                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2562                 // the default scenario build
2563                 if (os != 'Windows_NT') {
2564                     return false
2565                 }
2566                 // Only x64 for now
2567                 if (architecture != 'x64') {
2568                     return false
2569                 }
2570                 // Release only
2571                 if (configuration != 'Release') {
2572                     return false
2573                 }
2574                 break
2575             case 'jitdiff':
2576                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2577                     return false
2578                 }
2579                 if (architecture != 'x64') {
2580                     return false
2581                 }
2582                 if (configuration != 'Checked') {
2583                     return false
2584                 }
2585                 break
2586             case 'longgc':
2587             case 'gcsimulator':
2588                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2589                     return false
2590                 }
2591                 if (architecture != 'x64') {
2592                     return false
2593                 }
2594                 if (configuration != 'Release') {
2595                     return false
2596                 }
2597                 break
2598             case 'gc_reliability_framework':
2599             case 'standalone_gc':
2600                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2601                     return false
2602                 }
2603
2604                 if (architecture != 'x64') {
2605                     return false
2606                 }
2607
2608                 if (configuration != 'Release' && configuration != 'Checked') {
2609                     return false
2610                 }
2611                 break
2612             // We only run Windows and Ubuntu x64 Checked for formatting right now
2613             case 'formatting':
2614                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2615                     return false
2616                 }
2617                 if (architecture != 'x64') {
2618                     return false
2619                 }
2620                 if (configuration != 'Checked') {
2621                     return false
2622                 }
2623                 break
2624             case 'illink':
2625                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2626                     return false
2627                 }
2628                 if (architecture != 'x64' && architecture != 'x86') {
2629                     return false
2630                 }
2631                 break
2632             case 'normal':
2633                 // Nothing skipped
2634                 break
2635             case 'innerloop':
2636                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2637                     return false
2638                 }
2639                 break
2640             default:
2641                 println("Unknown scenario: ${scenario}")
2642                 assert false
2643                 break
2644         }
2645     }
2646
2647     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2648     // runtime behavior, not compile-time behavior, are not interesting.
2649     switch (architecture) {
2650         case 'x86_arm_altjit':
2651         case 'x64_arm64_altjit':
2652             if (isGCStressRelatedTesting(scenario)) {
2653                 return false
2654             }
2655             break
2656         default:
2657             break
2658     }
2659
2660     // The job was not filtered out, so we should generate it!
2661     return true
2662 }
2663
2664 Constants.allScenarios.each { scenario ->
2665     [true, false].each { isPR ->
2666         Constants.architectureList.each { architecture ->
2667             Constants.configurationList.each { configuration ->
2668                 Constants.osList.each { os ->
2669                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2670                     // and reset the os to Windows_NT
2671                     def isBuildOnly = false
2672                     if (os == 'Windows_NT_BuildOnly') {
2673                         isBuildOnly = true
2674                         os = 'Windows_NT'
2675                     }
2676
2677                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2678                         return
2679                     }
2680
2681                     // Calculate names
2682                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2683                     def folderName = getJobFolder(scenario)
2684
2685                     // Create the new job
2686                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2687                     addToViews(newJob, isPR, architecture, os)
2688
2689                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2690
2691                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2692                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2693                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2694
2695                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2696                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2697                     // artifacts produced.
2698                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2699                         // Define the Windows Tests and Corefx build job names
2700                         def lowerConfiguration = configuration.toLowerCase()
2701                         def WindowsTestsName = projectFolder + '/' +
2702                                                Utilities.getFullJobName(project,
2703                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2704                                                                         false)
2705                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2706                                            Utilities.getFolderName(branch)
2707
2708                         def arm_abi = 'arm'
2709                         def corefx_os = 'linux'
2710                         if (os == 'Tizen') {
2711                             arm_abi = 'armel'
2712                             corefx_os = 'tizen'
2713                         }
2714
2715                         // Let's use release CoreFX to test checked CoreCLR,
2716                         // because we do not generate checked CoreFX in CoreFX CI yet.
2717                         def corefx_lowerConfiguration = lowerConfiguration
2718                         if (lowerConfiguration == 'checked') {
2719                             corefx_lowerConfiguration = 'release'
2720                         }
2721
2722                         // Copy the Windows test binaries and the Corefx build binaries
2723                         newJob.with {
2724                             steps {
2725                                 copyArtifacts(WindowsTestsName) {
2726                                     includePatterns('bin/tests/tests.zip')
2727                                     buildSelector {
2728                                         latestSuccessful(true)
2729                                     }
2730                                 }
2731                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2732                                     includePatterns('bin/build.tar.gz')
2733                                     buildSelector {
2734                                         latestSuccessful(true)
2735                                     }
2736                                 }
2737                             } // steps
2738                         } // newJob.with
2739                     }
2740
2741                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2742
2743                     newJob.with {
2744                         steps {
2745                             if (os == 'Windows_NT') {
2746                                 buildCommands.each { buildCommand ->
2747                                     batchFile(buildCommand)
2748                                 }
2749                             }
2750                             else {
2751                                 buildCommands.each { buildCommand ->
2752                                     shell(buildCommand)
2753                                 }
2754                             }
2755                         } // steps
2756                     } // newJob.with
2757
2758                 } // os
2759             } // configuration
2760         } // architecture
2761     } // isPR
2762 } // scenario
2763
2764 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2765 // Returns the newly created job.
2766 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2767 {
2768     def osGroup = getOSGroup(os)
2769     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2770
2771     def jobFolder = getJobFolder(scenario)
2772     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2773         parameters {
2774             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2775         }
2776
2777         steps {
2778             // Set up the copies
2779
2780             // Coreclr build we are trying to test
2781             //
2782             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2783
2784             copyArtifacts(inputCoreCLRBuildName) {
2785                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2786                 buildSelector {
2787                     buildNumber('${CORECLR_BUILD}')
2788                 }
2789             }
2790
2791             if (isCoreFxScenario(scenario)) {
2792
2793                 // Only arm supported for corefx testing now.
2794                 assert architecture == 'arm'
2795
2796                 // Unzip CoreFx runtime
2797                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2798
2799                 // Unzip CoreFx tests.
2800                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2801
2802                 // Add the script to run the corefx tests
2803                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2804                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2805                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2806                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2807
2808             } else { // !isCoreFxScenario(scenario)
2809
2810                 // Unzip tests.
2811                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2812
2813                 def buildCommands = ""
2814
2815                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2816                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2817                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2818
2819                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2820                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2821
2822                 // For all jobs
2823                 addEnvVariable("CORE_ROOT", coreRootLocation)
2824                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2825                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2826
2827                 // ARM legacy backend; this is an altjit.
2828                 if (architecture == 'armlb') {
2829                     addEnvVariable("COMPlus_AltJit", "*")
2830                     addEnvVariable("COMPlus_AltJitNgen", "*")
2831                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2832                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2833                 }
2834
2835                 // If we are running a stress mode, we'll set those variables as well
2836                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2837                     def stressValues = null
2838                     if (isJitStressScenario(scenario)) {
2839                         stressValues = Constants.jitStressModeScenarios[scenario]
2840                     }
2841                     else {
2842                         stressValues = Constants.r2rStressScenarios[scenario]
2843                     }
2844
2845                     stressValues.each { key, value ->
2846                         addEnvVariable(key, value)
2847                     }
2848                 }
2849
2850                 if (isR2RScenario(scenario)) {
2851                     // Crossgen the framework assemblies.
2852                     buildCommands += """
2853 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2854 @goto skip_PrecompileAssembly
2855
2856 :PrecompileAssembly
2857 @REM Skip mscorlib since it is already precompiled.
2858 @if /I "%3" == "mscorlib.dll" exit /b 0
2859 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2860
2861 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2862 @if "%errorlevel%" == "-2146230517" (
2863     echo %2 is not a managed assembly.
2864 ) else if "%errorlevel%" == "-2146234344" (
2865     echo %2 is not a managed assembly.
2866 ) else if %errorlevel% neq 0 (
2867     echo Unable to precompile %2
2868 ) else (
2869     echo Precompiled %2
2870 )
2871 @exit /b 0
2872
2873 :skip_PrecompileAssembly
2874 """
2875
2876                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2877                     // crossgen on tests before running them.
2878                     addEnvVariable("RunCrossGen", "true")
2879                 } // isR2RScenario(scenario)
2880
2881                 // Create the smarty command
2882                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2883                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2884                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2885                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2886
2887                 if (architecture == 'armlb') {
2888                     addExclude("LEGACYJIT_FAIL")
2889                 }
2890
2891                 // Exclude tests based on scenario.
2892                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2893                     addArchSpecificExclude(architecture, excludeTag)
2894                 }
2895
2896                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2897                 if (scenario == 'innerloop') {
2898                     addExclude("pri1")
2899                 }
2900
2901                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2902                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2903                 addExclude("LONG_RUNNING")
2904
2905                 smartyCommand += "/lstFile Tests.lst"
2906
2907                 def testListArch = [
2908                     'arm64': 'arm64',
2909                     'arm': 'arm',
2910                     'armlb': 'arm'
2911                 ]
2912
2913                 def archLocation = testListArch[architecture]
2914
2915                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2916                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2917                 addCommand("${smartyCommand}")
2918
2919                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2920                 // However, we also need to remove all the variables that were set during this batch file, so we
2921                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2922                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2923                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2924                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2925                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2926                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2927
2928                 // ZIP up the smarty output, no matter what the smarty result.
2929                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2930
2931                 addCommand("echo %errorlevel%")
2932                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2933
2934                 // Use the smarty errorlevel as the script errorlevel.
2935                 addCommand("exit /b %__save_smarty_errorlevel%")
2936
2937                 batchFile(buildCommands)
2938             } // non-corefx testing
2939         } // steps
2940     } // job
2941
2942     if (!isCoreFxScenario(scenario)) {
2943         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2944
2945         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2946         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2947         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2948         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2949     }
2950
2951     return newJob
2952 }
2953
2954 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2955 // E.g., non-Windows tests.
2956 // Returns the newly created job.
2957 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2958 {
2959     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2960
2961     def osGroup = getOSGroup(os)
2962     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2963
2964     def testOpts = ''
2965     def useServerGC = false
2966
2967     // Enable Server GC for Ubuntu PR builds
2968     // REVIEW: why? Does this apply to all architectures? Why only PR?
2969     if (os == 'Ubuntu' && isPR) {
2970         testOpts += ' --useServerGC'
2971         useServerGC = true
2972     }
2973
2974     if (isR2RScenario(scenario)) {
2975
2976         testOpts += ' --crossgen --runcrossgentests'
2977
2978         if (scenario == 'r2r_jitstress1') {
2979             testOpts += ' --jitstress=1'
2980         }
2981         else if (scenario == 'r2r_jitstress2') {
2982             testOpts += ' --jitstress=2'
2983         }
2984         else if (scenario == 'r2r_jitstressregs1') {
2985             testOpts += ' --jitstressregs=1'
2986         }
2987         else if (scenario == 'r2r_jitstressregs2') {
2988             testOpts += ' --jitstressregs=2'
2989         }
2990         else if (scenario == 'r2r_jitstressregs3') {
2991             testOpts += ' --jitstressregs=3'
2992         }
2993         else if (scenario == 'r2r_jitstressregs4') {
2994             testOpts += ' --jitstressregs=4'
2995         }
2996         else if (scenario == 'r2r_jitstressregs8') {
2997             testOpts += ' --jitstressregs=8'
2998         }
2999         else if (scenario == 'r2r_jitstressregs0x10') {
3000             testOpts += ' --jitstressregs=0x10'
3001         }
3002         else if (scenario == 'r2r_jitstressregs0x80') {
3003             testOpts += ' --jitstressregs=0x80'
3004         }
3005         else if (scenario == 'r2r_jitstressregs0x1000') {
3006             testOpts += ' --jitstressregs=0x1000'
3007         }
3008         else if (scenario == 'r2r_jitminopts') {
3009             testOpts += ' --jitminopts'
3010         }
3011         else if (scenario == 'r2r_jitforcerelocs') {
3012             testOpts += ' --jitforcerelocs'
3013         }
3014         else if (scenario == 'r2r_gcstress15') {
3015             testOpts += ' --gcstresslevel=0xF'
3016         }
3017     }
3018     else if (scenario == 'jitdiff') {
3019         testOpts += ' --jitdisasm --crossgen'
3020     }
3021     else if (scenario == 'illink') {
3022         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3023     }
3024     else if (isLongGc(scenario)) {
3025         // Long GC tests behave very poorly when they are not
3026         // the only test running (many of them allocate until OOM).
3027         testOpts += ' --sequential'
3028
3029         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3030         // for running long GC and GCSimulator tests, respectively. We don't use them
3031         // here because using a playlist file produces much more readable output on the CI machines
3032         // and reduces running time.
3033         //
3034         // The Long GC playlist contains all of the tests that are
3035         // going to be run. The GCSimulator playlist contains all of
3036         // the GC simulator tests.
3037         if (scenario == 'longgc') {
3038             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3039         }
3040         else if (scenario == 'gcsimulator') {
3041             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3042         }
3043     }
3044     else if (isGcReliabilityFramework(scenario)) {
3045         testOpts += ' --build-overlay-only'
3046     }
3047     else if (scenario == 'standalone_gc') {
3048         if (osGroup == 'OSX') {
3049             testOpts += ' --gcname=libclrgc.dylib'
3050         }
3051         else if (osGroup == 'Linux') {
3052             testOpts += ' --gcname=libclrgc.so'
3053         }
3054         else {
3055             println("Unexpected OS group: ${osGroup} for os ${os}")
3056             assert false
3057         }
3058     }
3059
3060     def jobFolder = getJobFolder(scenario)
3061     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3062         parameters {
3063             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3064             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3065         }
3066
3067         steps {
3068             // Set up the copies
3069
3070             // Coreclr build containing the tests and mscorlib
3071             // pri1 jobs still need to copy windows_nt built tests
3072             assert inputTestsBuildName != null
3073             copyArtifacts(inputTestsBuildName) {
3074                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3075                 buildSelector {
3076                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3077                 }
3078             }
3079
3080             // Coreclr build we are trying to test
3081             //
3082             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3083
3084             copyArtifacts(inputCoreCLRBuildName) {
3085                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3086                 buildSelector {
3087                     buildNumber('${CORECLR_BUILD}')
3088                 }
3089             }
3090
3091             if (isUbuntuArmJob) {
3092                 // Add some useful information to the log file. Ignore return codes.
3093                 shell("uname -a || true")
3094             }
3095
3096             if (architecture == 'arm64') {
3097                 shell("mkdir -p ./bin/CoreFxBinDir")
3098                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3099                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3100             }
3101             else if (architecture == 'x86') {
3102                 shell("mkdir ./bin/CoreFxNative")
3103
3104                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3105
3106                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3107                     includePatterns('bin/build.tar.gz')
3108                     targetDirectory('bin/CoreFxNative')
3109                     buildSelector {
3110                         latestSuccessful(true)
3111                     }
3112                 }
3113
3114                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3115             }
3116
3117             // Unzip the tests first.  Exit with 0
3118             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3119             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3120
3121             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3122             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3123             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3124             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3125             // copied correctly.
3126             if (isUbuntuArmJob) {
3127                 def lowerConfiguration = configuration.toLowerCase()
3128                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3129                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3130             }
3131             else {
3132                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3133             }
3134
3135             // Execute the tests
3136             def runDocker = isNeedDocker(architecture, os, false)
3137             def dockerPrefix = ""
3138             def dockerCmd = ""
3139             if (runDocker) {
3140                 def dockerImage = getDockerImageName(architecture, os, false)
3141                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3142                 dockerCmd = dockerPrefix + "${dockerImage} "
3143             }
3144
3145             // If we are running a stress mode, we'll set those variables first
3146             if (isJitStressScenario(scenario)) {
3147                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3148                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3149                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3150                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3151                 shell("${envScriptCmds}")
3152                 testOpts += " --test-env=${scriptFileName}"
3153             }
3154
3155             // TODO: how to handle GCStress-related testing for Ubuntu/arm?
3156             if (isGCStressRelatedTesting(scenario)) {
3157                 shell('./init-tools.sh')
3158             }
3159
3160             def runScript = ""
3161             if (isUbuntuArmJob) {
3162                 // Use 'runtesttilstable.sh' to rerun failing tests (in sequential mode);
3163                 // there are many tests that pass on rerun (currently), and we don't want
3164                 // that flakiness to affect overall test job robustness.
3165                 runScript = "${dockerCmd}./tests/runtesttilstable.sh"
3166             } else {
3167                 runScript = "${dockerCmd}./tests/runtest.sh"
3168             }
3169
3170             shell("""\
3171 ${runScript} \\
3172     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3173     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3174     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3175     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3176
3177             if (isGcReliabilityFramework(scenario)) {
3178                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3179                 if (useServerGC) {
3180                     if (runDocker) {
3181                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3182                     }
3183                     else {
3184                         shell("export COMPlus_gcServer=1")
3185                     }
3186                 }
3187
3188                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3189             }
3190         } // steps
3191     } // job
3192
3193     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3194     if (os in ['Ubuntu']) {
3195         SummaryBuilder summaries = new SummaryBuilder()
3196         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3197         summaries.emit(newJob)
3198     }
3199
3200     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3201     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3202
3203     return newJob
3204 }
3205
3206 // Create a test job that will be used by a flow job.
3207 // Returns the newly created job.
3208 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3209 {
3210     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3211
3212     def newJob = null
3213     if (windowsArmJob) {
3214         assert inputTestsBuildName == null
3215         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3216     } else {
3217         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3218     }
3219
3220     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3221
3222     addToViews(newJob, isPR, architecture, os)
3223
3224     if (scenario == 'jitdiff') {
3225         def osGroup = getOSGroup(os)
3226         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3227     }
3228
3229     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3230     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3231
3232     return newJob
3233 }
3234
3235 // Create a flow job to tie together a build job with the given test job.
3236 // Returns the new flow job.
3237 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3238 {
3239     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3240     // Linux CoreCLR test
3241     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3242     def jobFolder = getJobFolder(scenario)
3243
3244     def newFlowJob = null
3245
3246     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3247     if (windowsArmJob) {
3248
3249         assert inputTestsBuildName == null
3250
3251         // For Windows arm jobs there is no reason to build a parallel test job.
3252         // The product build supports building and archiving the tests.
3253
3254         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3255                         buildFlow("""\
3256 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3257
3258 // And then build the test build
3259 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3260 """)
3261         }
3262         JobReport.Report.addReference(inputCoreCLRBuildName)
3263         JobReport.Report.addReference(fullTestJobName)
3264     }
3265     else {
3266         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3267                         buildFlow("""\
3268 // Build the input jobs in parallel
3269 parallel (
3270 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3271 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3272 )
3273
3274 // And then build the test build
3275 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3276                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3277 """)
3278         }
3279         JobReport.Report.addReference(inputCoreCLRBuildName)
3280         JobReport.Report.addReference(inputTestsBuildName)
3281         JobReport.Report.addReference(fullTestJobName)
3282     }
3283
3284     addToViews(newFlowJob, isPR, architecture, os)
3285
3286     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3287
3288     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3289     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3290
3291     return newFlowJob
3292 }
3293
3294 // Determine if we should generate a flow job for the given parameters.
3295 // Returns true if the job should be generated.
3296 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3297 {
3298     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3299     // All other scenarios do Pri-1 testing.
3300     if (scenario == 'innerloop' && !isPR) {
3301         return false
3302     }
3303
3304     // Filter based on OS and architecture.
3305
3306     switch (architecture) {
3307         case 'arm64':
3308             if (os != "Ubuntu" && os != "Windows_NT") {
3309                 return false
3310             }
3311             break
3312         case 'armlb':
3313             if (os != 'Windows_NT') {
3314                 return false
3315             }
3316             // Do not create armlb windows jobs.
3317             return false
3318         case 'arm':
3319             if (os != "Ubuntu" && os != "Windows_NT") {
3320                 return false
3321             }
3322             break
3323         case 'x86':
3324             if (os != "Ubuntu") {
3325                 return false
3326             }
3327             break
3328         case 'x64':
3329             if (!(os in Constants.crossList)) {
3330                 return false
3331             }
3332             if (os == "Windows_NT") {
3333                 return false
3334             }
3335             break
3336         case 'armem':
3337         case 'x86_arm_altjit':
3338         case 'x64_arm64_altjit':
3339             // No flow jobs
3340             return false
3341         default:
3342             println("Unknown architecture: ${architecture}")
3343             assert false
3344             break
3345     }
3346
3347     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3348
3349     // Filter based on scenario in OS.
3350
3351     if (os == 'Windows_NT') {
3352         if (!isArmWindowsScenario(scenario)) {
3353             return false
3354         }
3355     }
3356     else {
3357         // Non-Windows
3358         if (architecture == 'arm64') {
3359             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3360                 return false
3361             }
3362         }
3363         else if (architecture == 'arm') {
3364             if (!(scenario in Constants.validLinuxArmScenarios)) {
3365                 return false
3366             }
3367         }
3368         else if (architecture == 'x86') {
3369             // Linux/x86 only want innerloop and default test
3370             if (!isNormalOrInnerloop) {
3371                 return false
3372             }
3373         }
3374     }
3375
3376     // For CentOS, we only want Checked/Release builds.
3377     if (os == 'CentOS7.1') {
3378         if (configuration != 'Checked' && configuration != 'Release') {
3379             return false
3380         }
3381         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3382             return false
3383         }
3384     }
3385
3386     // For RedHat and Debian, we only do Release builds.
3387     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3388         if (configuration != 'Release') {
3389             return false
3390         }
3391         if (!isNormalOrInnerloop) {
3392             return false
3393         }
3394     }
3395
3396     // Next, filter based on scenario.
3397
3398     if (isJitStressScenario(scenario)) {
3399         if (configuration != 'Checked') {
3400             return false
3401         }
3402
3403         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3404         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3405             return false
3406         }
3407     }
3408     else if (isR2RBaselineScenario(scenario)) {
3409         if (configuration != 'Checked' && configuration != 'Release') {
3410             return false
3411         }
3412     }
3413     else if (isR2RStressScenario(scenario)) {
3414         if (configuration != 'Checked') {
3415             return false
3416         }
3417     }
3418     else {
3419         // Skip scenarios
3420         switch (scenario) {
3421             case 'ilrt':
3422             case 'longgc':
3423             case 'gcsimulator':
3424                 // Long GC tests take a long time on non-Release builds
3425                 // ilrt is also Release only
3426                 if (configuration != 'Release') {
3427                     return false
3428                 }
3429                 break
3430
3431             case 'jitdiff':
3432                 if (configuration != 'Checked') {
3433                     return false
3434                 }
3435                 break
3436
3437             case 'gc_reliability_framework':
3438             case 'standalone_gc':
3439                 if (configuration != 'Release' && configuration != 'Checked') {
3440                     return false
3441                 }
3442                 break
3443
3444             case 'formatting':
3445                 return false
3446             case 'illink':
3447                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3448                     return false
3449                 }
3450                 break
3451
3452             case 'normal':
3453                 // Nothing skipped
3454                 break
3455
3456             case 'innerloop':
3457                 // Nothing skipped
3458                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3459                     return false
3460                 }
3461                 break
3462
3463             default:
3464                 println("Unknown scenario: ${scenario}")
3465                 assert false
3466                 break
3467         }
3468     }
3469
3470     // The job was not filtered out, so we should generate it!
3471     return true
3472 }
3473
3474 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3475 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3476 Constants.allScenarios.each { scenario ->
3477     [true, false].each { isPR ->
3478         Constants.architectureList.each { architecture ->
3479             Constants.configurationList.each { configuration ->
3480                 Constants.osList.each { os ->
3481
3482                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3483                         return
3484                     }
3485
3486                     // Figure out the job name of the CoreCLR build the test will depend on.
3487
3488                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3489                     def inputCoreCLRBuildIsBuildOnly = false
3490                     if (isCoreFxScenario(scenario)) {
3491                         // Every CoreFx test depends on its own unique build.
3492                         inputCoreCLRBuildScenario = scenario
3493                         inputCoreCLRBuildIsBuildOnly = true
3494                     }
3495                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3496                     def inputCoreCLRBuildName = projectFolder + '/' +
3497                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3498
3499                     // Figure out the name of the build job that the test job will depend on.
3500                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3501                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3502
3503                     def inputTestsBuildName = null
3504
3505                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3506                     if (!windowsArmJob) {
3507                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3508
3509                         def inputTestsBuildArch = architecture
3510                         if (architecture == "arm64") {
3511                             // Use the x64 test build for arm64 unix
3512                             inputTestsBuildArch = "x64"
3513                         }
3514                         else if (architecture == "arm") {
3515                             // Use the x86 test build for arm unix
3516                             inputTestsBuildArch = "x86"
3517                         }
3518
3519                         def inputTestsBuildIsBuildOnly = true
3520
3521                         inputTestsBuildName = projectFolder + '/' +
3522                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3523                     }
3524
3525                     // =============================================================================================
3526                     // Create the test job
3527                     // =============================================================================================
3528
3529                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3530
3531                     // =============================================================================================
3532                     // Create a build flow to join together the build and tests required to run this test.
3533                     // =============================================================================================
3534
3535                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3536                         // Do not create the flow job for RHEL jobs.
3537                         return
3538                     }
3539
3540                     def fullTestJobName = projectFolder + '/' + testJob.name
3541                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3542
3543                 } // os
3544             } // configuration
3545         } // architecture
3546     } // isPR
3547 } // scenario
3548
3549 JobReport.Report.generateJobReport(out)
3550
3551 // Make the call to generate the help job
3552 Utilities.createHelperJob(this, project, branch,
3553     "Welcome to the ${project} Repository",  // This is prepended to the help message
3554     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3555
3556 Utilities.addCROSSCheck(this, project, branch)