netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink']
 155
 156     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 157
 158     // Valid PR trigger combinations.
 159     def static prTriggeredValidInnerLoopCombos = [
 160         'Windows_NT': [
 161             'x64': [
 162                 'Checked'
 163             ],
 164             'x86': [
 165                 'Checked',
 166                 'Release'
 167             ],
 168             'arm': [
 169                 'Checked',
 170             ],
 171             'arm64': [
 172                 'Checked'
 173             ],
 174             'armlb': [
 175                 'Checked'
 176             ]
 177         ],
 178         'Windows_NT_BuildOnly': [
 179             'x64': [
 180                 'Checked',
 181                 'Release'
 182             ],
 183             'x86': [
 184                 'Checked',
 185                 'Release'
 186             ],
 187             'arm': [
 188                 'Checked'
 189             ],
 190         ],
 191         'Ubuntu': [
 192             'x64': [
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug'
 197             ],
 198             'arm': [
 199                 'Checked'
 200             ]
 201         ],
 202         'CentOS7.1': [
 203             'x64': [
 204                 'Debug',
 205                 'Checked'
 206             ]
 207         ],
 208         'OSX10.12': [
 209             'x64': [
 210                 'Checked'
 211             ]
 212         ],
 213     ]
 214
 215     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 216     // to Tests.lst file categories to exclude.
 217     //
 218     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 219     // and with the same values, with some commented out, for easier maintenance.
 220     //
 221     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 222     //
 223     def static validArmWindowsScenarios = [
 224                'innerloop':                              [],
 225                'normal':                                 [],
 226                // 'ilrt'
 227                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 228                // 'longgc'
 229                // 'formatting'
 230                // 'gcsimulator'
 231                // 'jitdiff'
 232                // 'standalone_gc'
 233                // 'gc_reliability_framework'
 234                // 'illink'
 235                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 246                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 248                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 249                'tieredcompilation':                      [],
 250                'forcerelocs':                            [],
 251                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 270                // 'jitsse2only'                          // Only relevant to xarch
 271                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 272                // 'jitincompletehwintrinsic'
 273                // 'jitx86hwintrinsicnoavx'
 274                // 'jitx86hwintrinsicnoavx2'
 275                // 'jitx86hwintrinsicnosimd'
 276                // 'jitnox86hwintrinsic'
 277                'corefx_baseline':                        [],    // corefx tests don't use smarty
 278                'corefx_minopts':                         [],    // corefx tests don't use smarty
 279                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 280                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 281                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 282                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 283                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 284                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 285                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 286                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 290                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 291                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 292                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 293                'heapverify1':                            [],
 294                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 295                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 297                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 300
 301                //
 302                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 303                //
 304
 305                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 306                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 324     ]
 325
 326     def static validLinuxArm64Scenarios = [
 327                'innerloop',
 328                'normal',
 329                'r2r',
 330                'gcstress0x3',
 331                'gcstress0xc'
 332     ]
 333
 334     // Note: no GCStress-related scenario is enabled currently.
 335     def static validLinuxArmScenarios = [
 336                'innerloop',
 337                'normal',
 338                'r2r',
 339                'r2r_jitstress1',
 340                'r2r_jitstress2',
 341                'r2r_jitstressregs1',
 342                'r2r_jitstressregs2',
 343                'r2r_jitstressregs3',
 344                'r2r_jitstressregs4',
 345                'r2r_jitstressregs8',
 346                'r2r_jitstressregs0x10',
 347                'r2r_jitstressregs0x80',
 348                'r2r_jitstressregs0x1000',
 349                'r2r_jitminopts',
 350                'r2r_jitforcerelocs',
 351                // 'r2r_gcstress15',
 352                'minopts',
 353                'forcerelocs',
 354                'jitstress1',
 355                'jitstress2',
 356                'jitstressregs1',
 357                'jitstressregs2',
 358                'jitstressregs3',
 359                'jitstressregs4',
 360                'jitstressregs8',
 361                'jitstressregs0x10',
 362                'jitstressregs0x80',
 363                'jitstressregs0x1000',
 364                'jitstress2_jitstressregs1',
 365                'jitstress2_jitstressregs2',
 366                'jitstress2_jitstressregs3',
 367                'jitstress2_jitstressregs4',
 368                'jitstress2_jitstressregs8',
 369                'jitstress2_jitstressregs0x10',
 370                'jitstress2_jitstressregs0x80',
 371                'jitstress2_jitstressregs0x1000',
 372                'tailcallstress'
 373                // 'gcstress0x3',
 374                // 'gcstress0xc',
 375                // 'zapdisable',
 376                // 'heapverify1',
 377                // 'gcstress0xc_zapdisable',
 378                // 'gcstress0xc_zapdisable_jitstress2',
 379                // 'gcstress0xc_zapdisable_heapverify1',
 380                // 'gcstress0xc_jitstress1',
 381                // 'gcstress0xc_jitstress2',
 382                // 'gcstress0xc_minopts_heapverify1'
 383     ]
 384
 385     def static configurationList = ['Debug', 'Checked', 'Release']
 386
 387     // This is the set of architectures
 388     // Some of these are pseudo-architectures:
 389     //    armlb -- same as arm, but use the LEGACY_BACKEND JIT
 390     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 391     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 392     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 393     def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 394
 395     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 396     def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64']
 397 }
 398
 399 // **************************************************************
 400 // Create some specific views
 401 //
 402 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 403 // views based on a single regular expression. These views will be generated by adding a
 404 // specific set of jobs to them.
 405 //
 406 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 407 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 408 // those are required, don't add them (which simplifies the view pages, as well).
 409 // **************************************************************
 410
 411 class Views {
 412     def static MergeJobView = null
 413     def static PeriodicJobView = null
 414     def static ArchitectureViews = [:]
 415     def static OSViews = [:]
 416 }
 417
 418 // MergeJobView: include all jobs that execute when a PR change is merged.
 419 Views.MergeJobView = listView('Merge') {
 420     recurse()
 421     columns {
 422         status()
 423         weather()
 424         name()
 425         lastSuccess()
 426         lastFailure()
 427         lastDuration()
 428         buildButton()
 429     }
 430 }
 431
 432 // PeriodicJobView: include all jobs that execute on a schedule
 433 Views.PeriodicJobView = listView('Periodic') {
 434     recurse()
 435     columns {
 436         status()
 437         weather()
 438         name()
 439         lastSuccess()
 440         lastFailure()
 441         lastDuration()
 442         buildButton()
 443     }
 444 }
 445
 446 // Create a view for non-PR jobs for each architecture.
 447 Constants.architectureList.each { architecture ->
 448     Views.ArchitectureViews[architecture] = listView(architecture) {
 449         recurse()
 450         columns {
 451             status()
 452             weather()
 453             name()
 454             lastSuccess()
 455             lastFailure()
 456             lastDuration()
 457             buildButton()
 458         }
 459     }
 460 }
 461
 462 // Create a view for non-PR jobs for each OS.
 463 Constants.osList.each { os ->
 464     // Don't create one for the special 'Windows_NT_BuildOnly'
 465     if (os == 'Windows_NT_BuildOnly') {
 466         return
 467     }
 468     Views.OSViews[os] = listView(os) {
 469         recurse()
 470         columns {
 471             status()
 472             weather()
 473             name()
 474             lastSuccess()
 475             lastFailure()
 476             lastDuration()
 477             buildButton()
 478         }
 479     }
 480 }
 481
 482 def static addToMergeView(def job) {
 483     Views.MergeJobView.with {
 484         jobs {
 485             name(job.name)
 486         }
 487     }
 488 }
 489
 490 def static addToPeriodicView(def job) {
 491     Views.PeriodicJobView.with {
 492         jobs {
 493             name(job.name)
 494         }
 495     }
 496 }
 497
 498 def static addToViews(def job, def isPR, def architecture, def os) {
 499     if (isPR) {
 500         // No views want PR jobs currently.
 501         return
 502     }
 503
 504     // Add to architecture view.
 505     Views.ArchitectureViews[architecture].with {
 506         jobs {
 507             name(job.name)
 508         }
 509     }
 510
 511     // Add to OS view.
 512     Views.OSViews[os].with {
 513         jobs {
 514             name(job.name)
 515         }
 516     }
 517 }
 518
 519 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 520     addToPeriodicView(job)
 521     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 522 }
 523
 524 def static addGithubPushTriggerHelper(def job) {
 525     addToMergeView(job)
 526     Utilities.addGithubPushTrigger(job)
 527 }
 528
 529
 530 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 531     assert os instanceof String
 532     assert architecture instanceof String
 533
 534     def armArches = ['arm', 'armlb', 'armem', 'arm64']
 535     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 536
 537     if (!(architecture in armArches)) {
 538         assert options == null
 539         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 540
 541         return
 542     }
 543
 544     // This is an arm(64) job.
 545     //
 546     // There are several options.
 547     //
 548     // Windows_NT
 549     //
 550     // Arm32 (Build) -> latest-arm64
 551     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true
 552     // Arm32 (Test)  -> arm64-windows_nt
 553     //       |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false
 554     //
 555     // Arm64 (Build) -> latest-arm64
 556     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 557     // Arm64 (Test)  -> arm64-windows_nt
 558     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 559     //
 560     // Ubuntu
 561     //
 562     // Arm32 emulator (Build, Test) -> arm-cross-latest
 563     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 564     //
 565     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 566     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 567     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 568     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 569     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 570     //       |-> os == "Ubuntu" && (architecture == "arm")
 571     //
 572     // Arm64 (Build) -> arm64-cross-latest
 573     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 574     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 575     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 576     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 577     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 578
 579     // This has to be a arm arch
 580     assert architecture in armArches
 581     if (os == "Windows_NT") {
 582         // arm32/arm64 Windows jobs share the same machines for now
 583         def isBuild = options['use_arm64_build_machine'] == true
 584
 585         if (isBuild == true) {
 586             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 587         } else {
 588             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 589         }
 590     } else {
 591         assert os != 'Windows_NT'
 592         assert os in supportedArmLinuxOs
 593
 594         if (architecture == 'arm64') {
 595             if ((options != null) && (options['is_build_only'] == true)) {
 596                 // Arm64 Linux build machine
 597                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 598             } else {
 599                 // Arm64 Linux test machines
 600                 if ((options != null) && (options['large_pages'] == true)) {
 601                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 602                 } else {
 603                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 604                 }
 605             }
 606         }
 607         else if (architecture == 'armem') {
 608             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 609             // using Docker.
 610             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 611         }
 612         else {
 613             // arm Ubuntu on hardware.
 614             assert (architecture == 'arm') && (os == 'Ubuntu')
 615             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 616             def isBuild = (options != null) && (options['is_build_job'] == true)
 617             if (isFlow || isBuild) {
 618                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 619                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 620                 // arm hardware.
 621                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 622             } else {
 623                 // arm Ubuntu test machine
 624                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 625                 // Utilities.setMachineAffinity. Just add the machine affinity
 626                 // manually. We specify the Helix queue name here.
 627                 job.with {
 628                     label('ubuntu.1404.arm32.open')
 629                 }
 630             }
 631         }
 632     }
 633 }
 634
 635 // setJobMachineAffinity: compute the machine affinity options for a job,
 636 // then set the job with those affinity options.
 637 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 638 {
 639     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 640            (!isBuildJob && isTestJob  && !isFlowJob) ||
 641            (!isBuildJob && !isTestJob && isFlowJob)
 642
 643     def affinityOptions = null
 644     def affinityArchitecture = architecture
 645
 646     if (os == "Windows_NT") {
 647         if (architecture in Constants.armWindowsCrossArchitectureList) {
 648             if (isBuildJob) {
 649                 affinityOptions = [ "use_arm64_build_machine" : true ]
 650             } else if (isTestJob) {
 651                 affinityOptions = [ "use_arm64_build_machine" : false ]
 652             } else if (isFlowJob) {
 653                 // For the flow jobs set the machine affinity as x64
 654                 affinityArchitecture = 'x64'
 655             }
 656         }
 657     }
 658     else {
 659         if (architecture == 'arm64') {
 660             if (isBuildJob) {
 661                 affinityOptions = ['is_build_only': true]
 662             } else if (isTestJob) {
 663                 affinityOptions = [ "large_pages" : false ]
 664             }
 665         }
 666         else if (architecture == 'arm') {
 667             if (isBuildJob) {
 668                 affinityOptions = ['is_build_job': true]
 669             } else if (isFlowJob) {
 670                 affinityOptions = ['is_flow_job': true]
 671             }
 672         }
 673     }
 674
 675     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 676 }
 677
 678 def static isGCStressRelatedTesting(def scenario) {
 679     // The 'r2r_gcstress15' scenario is a basic scenario.
 680     // Detect it and make it a GCStress related.
 681     if (scenario == 'r2r_gcstress15')
 682     {
 683         return true;
 684     }
 685
 686     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 687     def scenarioName = scenario.toLowerCase()
 688     def isGCStressTesting = false
 689     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 690         if (k in gcStressTestEnvVars) {
 691             isGCStressTesting = true;
 692         }
 693     }
 694     return isGCStressTesting
 695 }
 696
 697 def static isCoreFxScenario(def scenario) {
 698     def corefx_prefix = 'corefx_'
 699     if (scenario.length() < corefx_prefix.length()) {
 700         return false
 701     }
 702     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 703 }
 704
 705 def static isR2RBaselineScenario(def scenario) {
 706     return (scenario == 'r2r')
 707 }
 708
 709 def static isR2RStressScenario(def scenario) {
 710     return Constants.r2rStressScenarios.containsKey(scenario)
 711 }
 712
 713 def static isR2RScenario(def scenario) {
 714     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 715 }
 716
 717 def static isJitStressScenario(def scenario) {
 718     return Constants.jitStressModeScenarios.containsKey(scenario)
 719 }
 720
 721 def static isLongGc(def scenario) {
 722     return (scenario == 'longgc' || scenario == 'gcsimulator')
 723 }
 724
 725 def static isJitDiff(def scenario) {
 726     return (scenario == 'jitdiff')
 727 }
 728
 729 def static isGcReliabilityFramework(def scenario) {
 730     return (scenario == 'gc_reliability_framework')
 731 }
 732
 733 def static isArmWindowsScenario(def scenario) {
 734     return Constants.validArmWindowsScenarios.containsKey(scenario)
 735 }
 736
 737 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 738     if (isBuildOnly == true) {
 739         os = 'Windows_NT_BuildOnly'
 740     }
 741
 742     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 743
 744     if (validOsPrTriggerArchConfigs == null) {
 745         return false
 746     }
 747
 748     if (validOsPrTriggerArchConfigs[architecture] != null) {
 749         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 750
 751         if (!(configuration in validOsPrTriggerConfigs)) {
 752             return false
 753         }
 754     } else {
 755         return false
 756     }
 757
 758     return true
 759 }
 760
 761 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 762     // 2 hours (120 minutes) is the default timeout
 763     def timeout = 120
 764     def innerLoop = (scenario == "innerloop")
 765
 766     if (!innerLoop) {
 767         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 768         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 769         timeout = 240
 770     }
 771
 772     if (!isBuildOnly) {
 773         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 774         if (isGCStressRelatedTesting(scenario)) {
 775             timeout = 4320
 776         }
 777         else if (isCoreFxScenario(scenario)) {
 778             timeout = 360
 779         }
 780         else if (isJitStressScenario(scenario)) {
 781             timeout = 300
 782         }
 783         else if (isR2RBaselineScenario(scenario)) {
 784             timeout = 240
 785         }
 786         else if (isLongGc(scenario)) {
 787             timeout = 1440
 788         }
 789         else if (isJitDiff(scenario)) {
 790             timeout = 240
 791         }
 792         else if (isGcReliabilityFramework(scenario)) {
 793             timeout = 1440
 794         }
 795         else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') {
 796             timeout = 240
 797         }
 798
 799         if (architecture == 'arm') {
 800             // ARM32 machines are particularly slow.
 801             timeout += 120
 802         }
 803     }
 804
 805     if (configuration == 'Debug') {
 806         // Debug runs can be very slow. Add an hour.
 807         timeout += 60
 808     }
 809
 810     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 811         // AltJit runs compile all methods twice.
 812         timeout *= 2
 813     }
 814
 815     // If we've changed the timeout from the default, set it in the job.
 816
 817     if (timeout != 120) {
 818         Utilities.setJobTimeout(newJob, timeout)
 819     }
 820 }
 821
 822 def static getJobFolder(def scenario) {
 823     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 824         return 'jitstress'
 825     }
 826     if (scenario == 'illink') {
 827         return 'illink'
 828     }
 829     return ''
 830 }
 831
 832 def static getStressModeDisplayName(def scenario) {
 833     def displayStr = ''
 834     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 835         def prefixLength = 'COMPlus_'.length()
 836         if (k.length() >= prefixLength) {
 837             def modeName = k.substring(prefixLength, k.length())
 838             displayStr += ' ' + modeName + '=' + v
 839         }
 840     }
 841
 842     if (isCoreFxScenario(scenario)) {
 843         displayStr = ('CoreFx ' + displayStr).trim()
 844     }
 845
 846     return displayStr
 847 }
 848
 849 def static getR2RDisplayName(def scenario) {
 850     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 851     def displayStr = scenario
 852     def prefixLength = 'r2r_'.length()
 853     if (displayStr.length() >= prefixLength) {
 854         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 855     } else if (scenario == 'r2r') {
 856         displayStr = "R2R"
 857     }
 858     return displayStr
 859 }
 860
 861 //
 862 // Functions to create an environment script.
 863 //      envScriptCreate -- initialize the script (call first)
 864 //      envScriptFinalize -- finalize the script (call last)
 865 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 866 //      envScriptAppendExistingScript -- append an existing script to the generated script
 867 //
 868 // Each script returns a string of commands. Concatenate all the strings together before
 869 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 870 //
 871
 872 // Initialize the environment setting script.
 873 def static envScriptCreate(def os, def stepScriptLocation) {
 874     def stepScript = ''
 875     if (os == 'Windows_NT') {
 876         stepScript += "echo Creating TestEnv script\r\n"
 877         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 878
 879         // Create at least an empty script.
 880         stepScript += "echo. > ${stepScriptLocation}\r\n"
 881     }
 882     else {
 883         stepScript += "echo Creating environment setting script\n"
 884         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 885     }
 886
 887     return stepScript
 888 }
 889
 890 // Generates the string for setting stress mode variables.
 891 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 892     def stepScript = ''
 893     if (os == 'Windows_NT') {
 894         stressModeVars.each{ k, v ->
 895             // Write out what we are writing to the script file
 896             stepScript += "echo Setting ${k}=${v}\r\n"
 897             // Write out the set itself to the script file`
 898             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 899         }
 900     }
 901     else {
 902         stressModeVars.each{ k, v ->
 903             // Write out what we are writing to the script file
 904             stepScript += "echo Setting ${k}=${v}\n"
 905             // Write out the set itself to the script file`
 906             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 907         }
 908     }
 909
 910     return stepScript
 911 }
 912
 913 // Append an existing script to an environment script.
 914 // Returns string of commands to do this.
 915 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 916     assert (os == 'Windows_NT')
 917     def stepScript = ''
 918
 919     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 920     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 921
 922     return stepScript
 923 }
 924
 925 // Finalize an environment setting script.
 926 // Returns string of commands to do this.
 927 def static envScriptFinalize(def os, def stepScriptLocation) {
 928     def stepScript = ''
 929
 930     if (os == 'Windows_NT') {
 931         // Display the resulting script. This is useful when looking at the output log file.
 932         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 933         stepScript += "type ${stepScriptLocation}\r\n"
 934     }
 935     else {
 936         stepScript += "chmod +x ${stepScriptLocation}\n"
 937     }
 938
 939     return stepScript
 940 }
 941
 942 def static isNeedDocker(def architecture, def os, def isBuild) {
 943     if (isBuild) {
 944         if (architecture == 'x86' && os == 'Ubuntu') {
 945             return true
 946         }
 947         else if (architecture == 'armem') {
 948             return true
 949         }
 950         else if (architecture == 'arm') {
 951             if (os == 'Ubuntu') {
 952                 return true
 953             }
 954         }
 955     }
 956     else {
 957         if (architecture == 'x86' && os == 'Ubuntu') {
 958             return true
 959         }
 960     }
 961     return false
 962 }
 963
 964 def static getDockerImageName(def architecture, def os, def isBuild) {
 965     // We must change some docker private images to official later
 966     if (isBuild) {
 967         if (architecture == 'x86' && os == 'Ubuntu') {
 968             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 969         }
 970         else if (architecture == 'armem') {
 971             if (os == 'Ubuntu') {
 972                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 973             }
 974             else if (os == 'Ubuntu16.04') {
 975                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 976             }
 977             else if (os == 'Tizen') {
 978                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 979             }
 980         }
 981         else if (architecture == 'arm') {
 982             if (os == 'Ubuntu') {
 983                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180323032140"
 984             }
 985         }
 986     }
 987     else {
 988         if (architecture == 'x86' && os == 'Ubuntu') {
 989             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 990         }
 991     }
 992     println("Unknown architecture to use docker: ${architecture} ${os}");
 993     assert false
 994 }
 995
 996
 997 // We have a limited amount of some hardware. For these, scale back the periodic testing we do.
 998 def static jobRequiresLimitedHardware(def architecture, def os) {
 999     if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) {
1000         // These test jobs require ARM64 hardware
1001         return true
1002     }
1003     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1004         // These test jobs require Linux/arm32 hardware
1005         return true
1006     }
1007     else {
1008         return false
1009     }
1010 }
1011
1012 // Calculates the name of the build job based on some typical parameters.
1013 //
1014 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1015     // If the architecture is x64, do not add that info into the build name.
1016     // Need to change around some systems and other builds to pick up the right builds
1017     // to do that.
1018
1019     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1020     if (isBuildOnly) {
1021         suffix += '_bld'
1022     }
1023     def baseName = ''
1024     switch (architecture) {
1025         case 'x64':
1026             if (scenario == 'normal') {
1027                 // For now we leave x64 off of the name for compatibility with other jobs
1028                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1029             }
1030             else if (scenario == 'formatting') {
1031                 // we don't care about the configuration for the formatting job. It runs all configs
1032                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1033             }
1034             else {
1035                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1036             }
1037             break
1038         case 'arm64':
1039             if (os.toLowerCase() == "windows_nt") {
1040                 // These are cross builds
1041                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1042             }
1043             else {
1044                 // Defaults to a small page size set of machines.
1045                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1046             }
1047             break
1048         case 'armem':
1049             // These are cross builds
1050             if (os == 'Tizen') {
1051                 // ABI: softfp
1052                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1053             }
1054             else {
1055                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1056             }
1057             break
1058         case 'armlb':
1059         case 'arm':
1060             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1061             break
1062         case 'x86':
1063         case 'x86_arm_altjit':
1064         case 'x64_arm64_altjit':
1065             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1066             break
1067         default:
1068             println("Unknown architecture: ${architecture}");
1069             assert false
1070             break
1071     }
1072
1073     return baseName + suffix
1074 }
1075
1076 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1077
1078     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
1079     if (os == 'Windows_NT') {
1080         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1081             if (!(branch in Constants.WindowsArm64Branches)) {
1082                 return
1083             }
1084         }
1085     }
1086
1087     // Check scenario.
1088     switch (scenario) {
1089         case 'innerloop':
1090             break
1091         case 'normal':
1092             switch (architecture) {
1093                 case 'x64':
1094                 case 'x86':
1095                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1096                         addPeriodicTriggerHelper(job, '@daily')
1097                     }
1098                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1099                         addGithubPushTriggerHelper(job)
1100                     }
1101                     break
1102                 case 'arm':
1103                     if (os == 'Windows_NT') {
1104                         addGithubPushTriggerHelper(job)
1105                     }
1106                     else {
1107                         // Currently no push triggers, with limited arm Linux hardware.
1108                         // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1109                         assert os == 'Ubuntu'
1110                         if (isFlowJob) {
1111                             addPeriodicTriggerHelper(job, '@daily')
1112                         }
1113                     }
1114                     break
1115                 case 'armem':
1116                 case 'armlb':
1117                 case 'x86_arm_altjit':
1118                 case 'x64_arm64_altjit':
1119                     addGithubPushTriggerHelper(job)
1120                     break
1121                 case 'arm64':
1122                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
1123                     addPeriodicTriggerHelper(job, "H H/4 * * *")
1124                     break
1125                 default:
1126                     println("Unknown architecture: ${architecture}");
1127                     assert false
1128                     break
1129             }
1130             break
1131         case 'r2r':
1132             assert !(os in bidailyCrossList)
1133             // r2r gets a push trigger for checked/release
1134             if (configuration == 'Checked' || configuration == 'Release') {
1135                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1136                 if (architecture == 'x64' && os != 'OSX10.12') {
1137                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1138                     if (isFlowJob || os == 'Windows_NT') {
1139                         addGithubPushTriggerHelper(job)
1140                     }
1141                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1142                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1143                     if (isFlowJob) {
1144                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1145                     }
1146                 }
1147                 // For x86, only add per-commit jobs for Windows
1148                 else if (architecture == 'x86') {
1149                     if (os == 'Windows_NT') {
1150                         addGithubPushTriggerHelper(job)
1151                     }
1152                 }
1153                 // arm64 r2r jobs should only run daily.
1154                 else if (architecture == 'arm64') {
1155                     if (os == 'Windows_NT') {
1156                         addPeriodicTriggerHelper(job, '@daily')
1157                     }
1158                 }
1159             }
1160             break
1161         case 'r2r_jitstress1':
1162         case 'r2r_jitstress2':
1163         case 'r2r_jitstressregs1':
1164         case 'r2r_jitstressregs2':
1165         case 'r2r_jitstressregs3':
1166         case 'r2r_jitstressregs4':
1167         case 'r2r_jitstressregs8':
1168         case 'r2r_jitstressregs0x10':
1169         case 'r2r_jitstressregs0x80':
1170         case 'r2r_jitstressregs0x1000':
1171         case 'r2r_jitminopts':
1172         case 'r2r_jitforcerelocs':
1173         case 'r2r_gcstress15':
1174             assert !(os in bidailyCrossList)
1175
1176             // GCStress=C is currently not supported on OS X
1177             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1178                 break
1179             }
1180
1181             // GC Stress 15 r2r gets a push trigger for checked/release
1182             if (configuration == 'Checked' || configuration == 'Release') {
1183                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1184                 if (architecture == 'x64') {
1185                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1186                     if (isFlowJob || os == 'Windows_NT') {
1187                         // Add a weekly periodic trigger
1188                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1189                     }
1190                 }
1191                 // For x86, only add per-commit jobs for Windows
1192                 else if (architecture == 'x86') {
1193                     if (os == 'Windows_NT') {
1194                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1195                     }
1196                 }
1197             }
1198             break
1199         case 'longgc':
1200             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1201             assert configuration == 'Release'
1202             assert architecture == 'x64'
1203             addPeriodicTriggerHelper(job, '@daily')
1204             // TODO: Add once external email sending is available again
1205             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1206             break
1207         case 'gcsimulator':
1208             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1209             assert configuration == 'Release'
1210             assert architecture == 'x64'
1211             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1212             // TODO: Add once external email sending is available again
1213             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1214             break
1215         case 'standalone_gc':
1216             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1217             assert (configuration == 'Release' || configuration == 'Checked')
1218             // TODO: Add once external email sending is available again
1219             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1220             addPeriodicTriggerHelper(job, '@daily')
1221             break
1222         case 'gc_reliability_framework':
1223             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1224             assert (configuration == 'Release' || configuration == 'Checked')
1225             // Only triggered by phrase.
1226             break
1227         case 'ilrt':
1228             assert !(os in bidailyCrossList)
1229             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1230             if (architecture == 'x64' && configuration == 'Release') {
1231                 // We don't expect to see a job generated except in these scenarios
1232                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1233                 if (isFlowJob || os == 'Windows_NT') {
1234                     addPeriodicTriggerHelper(job, '@daily')
1235                 }
1236             }
1237             break
1238         case 'jitdiff':
1239             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1240             assert configuration == 'Checked'
1241             assert (architecture == 'x64' || architecture == 'x86')
1242             addGithubPushTriggerHelper(job)
1243             break
1244         case 'formatting':
1245             assert (os == 'Windows_NT' || os == "Ubuntu")
1246             assert architecture == 'x64'
1247             addGithubPushTriggerHelper(job)
1248             break
1249         case 'jitstressregs1':
1250         case 'jitstressregs2':
1251         case 'jitstressregs3':
1252         case 'jitstressregs4':
1253         case 'jitstressregs8':
1254         case 'jitstressregs0x10':
1255         case 'jitstressregs0x80':
1256         case 'jitstressregs0x1000':
1257         case 'minopts':
1258         case 'forcerelocs':
1259         case 'jitstress1':
1260         case 'jitstress2':
1261         case 'jitstress2_jitstressregs1':
1262         case 'jitstress2_jitstressregs2':
1263         case 'jitstress2_jitstressregs3':
1264         case 'jitstress2_jitstressregs4':
1265         case 'jitstress2_jitstressregs8':
1266         case 'jitstress2_jitstressregs0x10':
1267         case 'jitstress2_jitstressregs0x80':
1268         case 'jitstress2_jitstressregs0x1000':
1269         case 'tailcallstress':
1270         case 'jitsse2only':
1271         case 'jitnosimd':
1272         case 'jitnox86hwintrinsic':
1273         case 'jitincompletehwintrinsic':
1274         case 'jitx86hwintrinsicnoavx':
1275         case 'jitx86hwintrinsicnoavx2':
1276         case 'jitx86hwintrinsicnosimd':
1277         case 'corefx_baseline':
1278         case 'corefx_minopts':
1279         case 'corefx_jitstress1':
1280         case 'corefx_jitstress2':
1281         case 'corefx_jitstressregs1':
1282         case 'corefx_jitstressregs2':
1283         case 'corefx_jitstressregs3':
1284         case 'corefx_jitstressregs4':
1285         case 'corefx_jitstressregs8':
1286         case 'corefx_jitstressregs0x10':
1287         case 'corefx_jitstressregs0x80':
1288         case 'corefx_jitstressregs0x1000':
1289         case 'zapdisable':
1290             if (os == 'CentOS7.1') {
1291                 break
1292             }
1293             if (os in bidailyCrossList) {
1294                 break
1295             }
1296             assert (os == 'Windows_NT') || (os in Constants.crossList)
1297             if (jobRequiresLimitedHardware(architecture, os)) {
1298                 addPeriodicTriggerHelper(job, '@weekly')
1299             }
1300             else {
1301                 addPeriodicTriggerHelper(job, '@daily')
1302             }
1303             break
1304         case 'heapverify1':
1305         case 'gcstress0x3':
1306             if (os == 'CentOS7.1') {
1307                 break
1308             }
1309             if (os in bidailyCrossList) {
1310                 break
1311             }
1312             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1313                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1314                 break
1315             }
1316             assert (os == 'Windows_NT') || (os in Constants.crossList)
1317             addPeriodicTriggerHelper(job, '@weekly')
1318             break
1319         case 'gcstress0xc':
1320         case 'gcstress0xc_zapdisable':
1321         case 'gcstress0xc_zapdisable_jitstress2':
1322         case 'gcstress0xc_zapdisable_heapverify1':
1323         case 'gcstress0xc_jitstress1':
1324         case 'gcstress0xc_jitstress2':
1325         case 'gcstress0xc_minopts_heapverify1':
1326             if (os == 'CentOS7.1') {
1327                 break
1328             }
1329             if (os == 'OSX10.12') {
1330                 // GCStress=C is currently not supported on OS X
1331                 break
1332             }
1333             if (os in bidailyCrossList) {
1334                 break
1335             }
1336             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1337                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1338                 break
1339             }
1340             assert (os == 'Windows_NT') || (os in Constants.crossList)
1341             addPeriodicTriggerHelper(job, '@weekly')
1342             break
1343
1344         case 'illink':
1345             // Testing on other operating systems TBD
1346             assert (os == 'Windows_NT' || os == 'Ubuntu')
1347             if (architecture == 'x64' || architecture == 'x86') {
1348                 if (configuration == 'Checked') {
1349                     addPeriodicTriggerHelper(job, '@daily')
1350                 }
1351             }
1352             break
1353
1354         case 'tieredcompilation':
1355         case 'corefx_tieredcompilation':
1356             // No periodic jobs just yet, still testing
1357             break
1358
1359         default:
1360             println("Unknown scenario: ${scenario}");
1361             assert false
1362             break
1363     }
1364     return
1365 }
1366
1367 // **************************
1368 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1369 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1370 // builds will do a couple extra steps.
1371 // **************************
1372
1373 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1374 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1375 // it's just used for internal builds
1376 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1377 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1378     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1379
1380     if (isWindowsBuildOnlyJob) {
1381         return
1382     }
1383
1384     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1385     // Non pull request builds.
1386     if (!isPR) {
1387         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1388         return
1389     }
1390
1391      def arm64Users = [
1392         'AndyAyersMS',
1393         'briansull',
1394         'BruceForstall',
1395         'CarolEidt',
1396         'cmckinsey',
1397         'echesakovMSFT',
1398         'erozenfeld',
1399         'janvorli',
1400         'jashook',
1401         'JosephTremoulet',
1402         'pgodeq',
1403         'russellhadley',
1404         'RussKeldorph',
1405         'sandreenko',
1406         'sdmaclea',
1407         'swaroop-sridhar',
1408         'jkotas',
1409         'markwilkie',
1410         'weshaggard'
1411     ]
1412
1413     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1414     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1415     def osGroup = getOSGroup(os)
1416     switch (architecture) {
1417         case 'x64': // editor brace matching: {
1418             if (scenario == 'formatting') {
1419                 assert configuration == 'Checked'
1420                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1421                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1422                 }
1423
1424                 break
1425             }
1426
1427             switch (os) {
1428                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1429                 case 'Debian8.4':
1430                 case 'RHEL7.2':
1431                     if (scenario == 'innerloop') {
1432                         assert !isFlowJob
1433                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1434                     }
1435                     else if (scenario == 'normal') {
1436                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1437                     }
1438                     break
1439
1440                 case 'Ubuntu16.04':
1441                     assert !isFlowJob
1442                     assert scenario != 'innerloop'
1443                     // Distinguish with the other architectures (arm and x86)
1444                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1445                     break
1446
1447                 case 'Fedora24':
1448                 case 'Ubuntu16.10':
1449                     assert !isFlowJob
1450                     assert scenario != 'innerloop'
1451                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1452                     break
1453
1454                 case 'Ubuntu':
1455                     if (scenario == 'illink') {
1456                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1457                         break
1458                     }
1459                     // fall through
1460
1461                 case 'OSX10.12':
1462                     // Triggers on the non-flow jobs aren't necessary here
1463                     // Corefx testing uses non-flow jobs.
1464                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1465                         break
1466                     }
1467                     switch (scenario) {
1468                         case 'innerloop':
1469                             // PR Triggered jobs. These jobs will run pri0 tests.
1470                             if (configuration == 'Checked') {
1471                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1472                             }
1473                             break
1474
1475                         case 'normal':
1476                             // OSX uses checked for default PR tests
1477                             if (configuration == 'Checked') {
1478                                 // Default trigger
1479                                 assert !job.name.contains("centos")
1480                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1481                             }
1482                             break
1483
1484                         case 'jitdiff':
1485                             if (configuration == 'Checked') {
1486                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1487                             }
1488                             break
1489
1490                         case 'ilrt':
1491                             if (configuration == 'Release') {
1492                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1493                             }
1494                             break
1495
1496                         case 'longgc':
1497                             if (configuration == 'Release') {
1498                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1499                             }
1500                             break
1501
1502                         case 'gcsimulator':
1503                             if (configuration == 'Release') {
1504                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1505                             }
1506                             break
1507
1508                         case 'standalone_gc':
1509                             if (configuration == 'Release' || configuration == 'Checked') {
1510                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1511                             }
1512                             break
1513
1514                         case 'gc_reliability_framework':
1515                             if (configuration == 'Release' || configuration == 'Checked') {
1516                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1517                             }
1518                             break
1519
1520                         default:
1521                             if (isJitStressScenario(scenario)) {
1522                                 def displayStr = getStressModeDisplayName(scenario)
1523                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1524                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1525                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1526                             }
1527                             else if (isR2RScenario(scenario)) {
1528                                 if (configuration == 'Release' || configuration == 'Checked') {
1529                                     def displayStr = getR2RDisplayName(scenario)
1530                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1531                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1532                                 }
1533                             }
1534                             else {
1535                                 println("Unknown scenario: ${scenario}");
1536                                 assert false
1537                             }
1538                             break
1539
1540                     }
1541                     break
1542
1543                 case 'CentOS7.1':
1544                     switch (scenario) {
1545                         case 'innerloop':
1546                             // CentOS uses checked for default PR tests while debug is build only
1547                             if (configuration == 'Debug') {
1548                                 // Default trigger
1549                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1550                             }
1551
1552                             // Make sure this is a flow job to get build and test.
1553                             if (configuration == 'Checked' && isFlowJob) {
1554                                 assert job.name.contains("flow")
1555                                 // Default trigger
1556                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1557                             }
1558                             break
1559
1560                         case 'normal':
1561                             // Make sure this is a flow job to get build and test.
1562                             if (configuration == 'Checked' && isFlowJob) {
1563                                 assert job.name.contains("flow")
1564                                 // Default trigger
1565                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1566                             }
1567                             break
1568
1569                         default:
1570                             if (isR2RScenario(scenario)) {
1571                                 if (configuration == 'Release' || configuration == 'Checked') {
1572                                     def displayStr = getR2RDisplayName(scenario)
1573                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1574                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1575                                 }
1576                             }
1577                             break
1578
1579                     }
1580                     break
1581
1582                 case 'Windows_NT':
1583                     switch (scenario) {
1584                         case 'innerloop':
1585                             // Default trigger
1586                             if (configuration == 'Checked' || configuration == 'Release') {
1587                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1588                             }
1589                             break
1590
1591                         case 'normal':
1592                             if (configuration == 'Checked') {
1593                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1594                             }
1595                             break
1596
1597                         case 'jitdiff':
1598                             if (configuration == 'Checked') {
1599                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1600                             }
1601                             break
1602
1603                         case 'ilrt':
1604                             if (configuration == 'Release') {
1605                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1606                             }
1607                             break
1608
1609                         case 'longgc':
1610                             if (configuration == 'Release') {
1611                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1612                             }
1613                             break
1614
1615                         case 'gcsimulator':
1616                             if (configuration == 'Release') {
1617                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1618                             }
1619                             break
1620
1621                         case 'standalone_gc':
1622                             if (configuration == 'Release' || configuration == 'Checked') {
1623                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1624                             }
1625                             break
1626
1627                         case 'gc_reliability_framework':
1628                             if (configuration == 'Release' || configuration == 'Checked') {
1629                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1630                             }
1631                             break
1632
1633                         case 'illink':
1634                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1635                             break
1636
1637                         default:
1638                             if (isJitStressScenario(scenario)) {
1639                                 def displayStr = getStressModeDisplayName(scenario)
1640                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1641                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1642                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1643                             }
1644                             else if (isR2RScenario(scenario)) {
1645                                 if (configuration == 'Release' || configuration == 'Checked') {
1646                                     def displayStr = getR2RDisplayName(scenario)
1647                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1648                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1649                                 }
1650                             }
1651                             else {
1652                                 println("Unknown scenario: ${scenario}");
1653                                 assert false
1654                             }
1655                             break
1656
1657                     }
1658                     break
1659
1660                 default:
1661                     println("Unknown os: ${os}");
1662                     assert false
1663                     break
1664
1665             }
1666
1667             break
1668         // editor brace matching: }
1669
1670         case 'armem': // editor brace matching: {
1671             job.with {
1672                 publishers {
1673                     azureVMAgentPostBuildAction {
1674                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1675                     }
1676                 }
1677             }
1678
1679             switch (os) {
1680                 case 'Ubuntu':
1681                 case 'Ubuntu16.04':
1682                     assert scenario != 'innerloop'
1683                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1684                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1685                     break
1686
1687                 case 'Tizen':
1688                     architecture = 'armel'
1689
1690                     assert scenario != 'innerloop'
1691                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1692                         "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1693                     break
1694             }
1695
1696             break
1697         // editor brace matching: }
1698
1699         case 'armlb':
1700         case 'arm': // editor brace matching: {
1701
1702             // Triggers on the non-flow jobs aren't necessary
1703             if (!isFlowJob) {
1704                 break
1705             }
1706
1707             // Set up a private trigger
1708             def contextString = "${os} ${architecture} Cross ${configuration}"
1709             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1710             if (scenario == 'innerloop') {
1711                 contextString += " Innerloop"
1712                 triggerString += "\\W+Innerloop"
1713             }
1714             else {
1715                 contextString += " ${scenario}"
1716                 triggerString += "\\W+${scenario}"
1717             }
1718
1719             if (configuration == 'Debug') {
1720                 contextString += " Build"
1721                 triggerString += "\\W+Build"
1722             } else {
1723                 contextString += " Build and Test"
1724                 triggerString += "\\W+Build and Test"
1725             }
1726
1727             triggerString += ".*"
1728
1729             switch (os) {
1730                 case 'Ubuntu':
1731                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1732                         break
1733                     }
1734
1735                     if (scenario == 'innerloop') {
1736                         if (configuration == 'Checked') {
1737                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1738                         }
1739                     }
1740                     else {
1741                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1742                     }
1743                     break
1744
1745                 case 'Windows_NT':
1746                     if (architecture == "armlb") {
1747                         // Disable armlb windows jobs
1748                         break
1749                     }
1750                     switch (scenario) {
1751                         case 'innerloop':
1752                             // Only Checked is an innerloop trigger.
1753                             if (configuration == 'Checked')
1754                             {
1755                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1756                             }
1757                             break
1758                         case 'normal':
1759                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1760                             break
1761                         default:
1762                             // Stress jobs will use this code path.
1763                             if (isArmWindowsScenario(scenario)) {
1764                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1765                             }
1766                             break
1767                     }
1768                     break
1769                 default:
1770                     println("NYI os: ${os}");
1771                     assert false
1772                     break
1773             }
1774             break
1775         // editor brace matching: }
1776         case 'arm64': // editor brace matching: {
1777             // Set up a private trigger
1778             def contextString = "${os} ${architecture} Cross ${configuration}"
1779             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1780
1781             if (scenario == 'innerloop') {
1782                 contextString += " Innerloop"
1783                 triggerString += "\\W+Innerloop"
1784             }
1785             else {
1786                 contextString += " ${scenario}"
1787                 triggerString += "\\W+${scenario}"
1788             }
1789
1790             if (configuration == 'Debug') {
1791                 contextString += " Build"
1792                 triggerString += "\\W+Build"
1793             } else {
1794                 contextString += " Build and Test"
1795                 triggerString += "\\W+Build and Test"
1796             }
1797
1798             triggerString += ".*"
1799
1800             switch (os) {
1801                 case 'Ubuntu':
1802                 case 'Ubuntu16.04':
1803                     switch (scenario) {
1804                         case 'innerloop':
1805                             if (configuration == 'Debug' && !isFlowJob) {
1806                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1807                             }
1808
1809                             break
1810                         case 'normal':
1811                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1812                             break
1813                         default:
1814                             if (isR2RScenario(scenario)) {
1815                                 if (configuration == 'Checked' || configuration == 'Release') {
1816                                     def displayStr = getR2RDisplayName(scenario)
1817                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1818                                 }
1819                             }
1820                             break
1821                     }
1822                     break
1823
1824                 case 'Windows_NT':
1825                     // Triggers on the non-flow jobs aren't necessary here
1826                     if (!isFlowJob) {
1827                         break
1828                     }
1829
1830                     assert isArmWindowsScenario(scenario)
1831                     switch (scenario) {
1832                         case 'innerloop':
1833                             if (configuration == 'Checked') {
1834                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1835                             }
1836
1837                             break
1838                         case 'normal':
1839                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1840                             break
1841                         default:
1842                             // Stress jobs will use this code path.
1843                             if (isArmWindowsScenario(scenario)) {
1844                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1845                             }
1846                             break
1847                     }
1848                     break
1849                 default:
1850                     println("NYI os: ${os}");
1851                     assert false
1852                     break
1853             }
1854             break
1855
1856         // editor brace matching: }
1857         case 'x86': // editor brace matching: {
1858             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1859             if (os == 'Ubuntu') {
1860                 // Triggers on the non-flow jobs aren't necessary here
1861                 if (!isFlowJob) {
1862                     break
1863                 }
1864
1865                 // on-demand only for ubuntu x86
1866                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1867                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1868                 break
1869
1870             }
1871             switch (scenario) {
1872                 case 'innerloop':
1873                     if (configuration == 'Checked' || configuration == 'Release') {
1874                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1875                     }
1876                     break
1877
1878                 case 'normal':
1879                     if (configuration == 'Checked') {
1880                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1881                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1882                     }
1883                     break
1884
1885                 case 'ilrt':
1886                     if (configuration == 'Release') {
1887                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1888                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1889                     }
1890                     break
1891
1892                 case 'longgc':
1893                     if (configuration == 'Release') {
1894                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1895                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1896                     }
1897                     break
1898
1899                 case 'gcsimulator':
1900                     if (configuration == 'Release') {
1901                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1902                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1903                     }
1904                     break
1905
1906                 case 'standalone_gc':
1907                     if (configuration == 'Release' || configuration == 'Checked') {
1908                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1909                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1910                     }
1911                     break
1912
1913                 case 'illink':
1914                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1915                     break
1916
1917                 default:
1918                     if (isJitStressScenario(scenario)) {
1919                         def displayStr = getStressModeDisplayName(scenario)
1920                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1921                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1922                     }
1923                     else if (isR2RScenario(scenario)) {
1924                         if (configuration == 'Release' || configuration == 'Checked') {
1925                             def displayStr = getR2RDisplayName(scenario)
1926                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1927                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1928                         }
1929                     }
1930                     else {
1931                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1932                         assert false
1933                     }
1934                     break
1935
1936             }
1937             break
1938
1939          // editor brace matching: }
1940         case 'x64_arm64_altjit':
1941         case 'x86_arm_altjit': // editor brace matching: {
1942             assert (os == 'Windows_NT')
1943             switch (scenario) {
1944                 case 'normal':
1945                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1946                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1947                     break
1948                 default:
1949                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1950                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1951                     break
1952             }
1953             break
1954
1955         // editor brace matching: }
1956         default:
1957             println("Unknown architecture: ${architecture}");
1958             assert false
1959             break
1960     }
1961 }
1962
1963 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1964     def buildCommands = []
1965     def osGroup = getOSGroup(os)
1966     def lowerConfiguration = configuration.toLowerCase()
1967
1968     def priority = '1'
1969     if (scenario == 'innerloop') {
1970         priority = '0'
1971     }
1972
1973     def doCoreFxTesting = isCoreFxScenario(scenario)
1974
1975     // Calculate the build steps, archival, and xunit results
1976     switch (os) {
1977         case 'Windows_NT': // editor brace matching: {
1978             switch (architecture) {
1979                 case 'x64':
1980                 case 'x86':
1981                 case 'x86_arm_altjit':
1982                 case 'x64_arm64_altjit':
1983                     def arch = architecture
1984                     def buildOpts = ''
1985                     if (architecture == 'x86_arm_altjit') {
1986                         arch = 'x86'
1987                     }
1988                     else if (architecture == 'x64_arm64_altjit') {
1989                         arch = 'x64'
1990                     }
1991
1992                     if (scenario == 'formatting') {
1993                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1994                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1995                         break
1996                     }
1997
1998                     if (scenario == 'illink') {
1999                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2000                     }
2001
2002                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2003                     if ((lowerConfiguration == 'release') &&
2004                         (scenario in Constants.basicScenarios) &&
2005                         (architecture != 'x86_arm_altjit') &&
2006                         (architecture != 'x64_arm64_altjit')) {
2007
2008                         buildOpts += ' -enforcepgo'
2009                     }
2010
2011                     if (doCoreFxTesting) {
2012                         buildOpts += ' skiptests';
2013                     } else {
2014                         buildOpts += " -priority=${priority}"
2015                     }
2016
2017                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2018                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2019                     // 35 characters long.
2020
2021                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2022
2023                     if (!isBuildOnly) {
2024                         def runtestArguments = ''
2025                         def testOpts = 'collectdumps'
2026
2027                         if (isR2RScenario(scenario)) {
2028
2029                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2030                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2031                             // to cause the tests to be crossgen'ed.
2032
2033                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2034                                 testOpts += ' crossgenaltjit protononjit.dll'
2035                             } else {
2036                                 testOpts += ' crossgen'
2037                             }
2038
2039                             testOpts += ' runcrossgentests'
2040                         }
2041                         else if (scenario == 'jitdiff') {
2042                             testOpts += ' jitdisasm crossgen'
2043                         }
2044                         else if (scenario == 'ilrt') {
2045                             testOpts += ' ilasmroundtrip'
2046                         }
2047                         else if (isLongGc(scenario)) {
2048                             testOpts += " ${scenario} sequential"
2049                         }
2050                         else if (scenario == 'standalone_gc') {
2051                             testOpts += ' gcname clrgc.dll'
2052                         }
2053                         else if (scenario == 'illink') {
2054                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2055                         }
2056
2057                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2058                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2059                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2060                         // which is read by the xunit harness.)
2061                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2062                         {
2063                             def timeout = 1800000
2064                             testOpts += " timeout ${timeout}"
2065                         }
2066
2067                         // If we are running a stress mode, we should write out the set of key
2068                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2069
2070                         def envScriptPath = ''
2071                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2072                             def buildCommandsStr = ''
2073                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2074                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2075
2076                             if (isJitStressScenario(scenario)) {
2077                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2078                             }
2079                             else if (isR2RStressScenario(scenario)) {
2080                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2081                             }
2082
2083                             if (architecture == 'x86_arm_altjit') {
2084                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2085                             }
2086                             else if (architecture == 'x64_arm64_altjit') {
2087                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2088                             }
2089
2090                             envScriptFinalize(os, envScriptPath)
2091
2092                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2093                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2094                             // into a single script.
2095                             buildCommands += buildCommandsStr
2096                         }
2097                         else if (architecture == 'x86_arm_altjit') {
2098                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2099                         }
2100                         else if (architecture == 'x64_arm64_altjit') {
2101                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2102                         }
2103                         if (envScriptPath != '') {
2104                             testOpts += " TestEnv ${envScriptPath}"
2105                         }
2106
2107                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2108
2109                         if (doCoreFxTesting) {
2110                             def workspaceRelativeFxRoot = "_/fx"
2111                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2112
2113                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
2114
2115                             // Archive and process (only) the test results
2116                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2117                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2118
2119                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2120                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2121                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2122                         }
2123                         else if (isGcReliabilityFramework(scenario)) {
2124                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2125                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2126                         }
2127                         else {
2128                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2129                         }
2130                     } // end if (!isBuildOnly)
2131
2132                     if (!doCoreFxTesting) {
2133                         // Run the rest of the build
2134                         // Build the mscorlib for the other OS's
2135                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2136                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2137
2138                         if (arch == 'x64') {
2139                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2140                         }
2141
2142                         // Zip up the tests directory so that we don't use so much space/time copying
2143                         // 10s of thousands of files around.
2144                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2145
2146                         if (!isJitStressScenario(scenario)) {
2147                             // For Windows, pull full test results and test drops for x86/x64.
2148                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2149                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2150                         }
2151
2152                         if (scenario == 'jitdiff') {
2153                             // retrieve jit-dasm output for base commit, and run jit-diff
2154                             if (!isBuildOnly) {
2155                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2156                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2157                             }
2158                         }
2159
2160                         if (!isBuildOnly) {
2161                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2162                         }
2163                     }
2164                     break
2165                 case 'armlb':
2166                 case 'arm':
2167                     assert isArmWindowsScenario(scenario)
2168
2169                     def buildArchitecture = 'arm'
2170
2171                     def buildOpts = ''
2172
2173                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
2174                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
2175                     if (architecture == 'armlb') {
2176                         buildOpts += ' -crossgenaltjit legacyjit.dll'
2177                     }
2178
2179                     if (doCoreFxTesting) {
2180                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2181                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2182                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2183                         //
2184                         // buildOpts += ' skiptests';
2185                         buildOpts += " -priority=0"
2186                     } else {
2187                         buildOpts += " -priority=${priority}"
2188                     }
2189
2190                     // This is now a build only job. Do not run tests. Use the flow job.
2191                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2192
2193                     if (doCoreFxTesting) {
2194                         assert isBuildOnly
2195                         assert architecture == 'arm'
2196
2197                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2198                         // to determine the correct matching corefx version git commit hash.
2199                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2200
2201                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2202                         def envScriptPath = ''
2203                         def buildCommandsStr = ''
2204                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2205                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2206                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2207                         envScriptFinalize(os, envScriptPath)
2208                         buildCommands += buildCommandsStr
2209
2210                         def workspaceRelativeFxRootLinux = "_/fx"
2211                         def workspaceRelativeFxRootWin = "_\\fx"
2212                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2213
2214                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath} -no_run_tests"
2215
2216                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2217                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2218                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2219
2220                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2221                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2222                     } else {
2223                         // Zip up the tests directory so that we don't use so much space/time copying
2224                         // 10s of thousands of files around.
2225                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2226
2227                         // Add archival.
2228                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2229                     }
2230                     break
2231                 case 'arm64':
2232                     assert isArmWindowsScenario(scenario)
2233
2234                     // This is now a build only job. Do not run tests. Use the flow job.
2235                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2236
2237                     // Zip up the tests directory so that we don't use so much space/time copying
2238                     // 10s of thousands of files around.
2239                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2240
2241                     // Add archival.
2242                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2243                     break
2244                 default:
2245                     println("Unknown architecture: ${architecture}");
2246                     assert false
2247                     break
2248             }
2249             break
2250         // end case 'Windows_NT'; editor brace matching: }
2251         case 'Ubuntu':
2252         case 'Ubuntu16.04':
2253         case 'Ubuntu16.10':
2254         case 'Debian8.4':
2255         case 'OSX10.12':
2256         case 'CentOS7.1':
2257         case 'RHEL7.2':
2258         case 'Tizen':
2259         case 'Fedora24': // editor brace matching: {
2260             switch (architecture) {
2261                 case 'x64':
2262                 case 'x86':
2263                     if (architecture == 'x86' && os == 'Ubuntu') {
2264                         // build and PAL test
2265                         def dockerImage = getDockerImageName(architecture, os, true)
2266                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2267                         dockerImage = getDockerImageName(architecture, os, false)
2268                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2269                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2270                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2271                         break
2272                     }
2273
2274                     if (scenario == 'formatting') {
2275                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2276                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2277                         break
2278                     }
2279
2280                     if (scenario == 'illink') {
2281                         assert(os == 'Ubuntu')
2282                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2283                     }
2284
2285                     if (!doCoreFxTesting) {
2286                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2287                         // only on supported OS platforms.
2288                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2289                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2290
2291                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2292                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2293
2294                         // Basic archiving of the build
2295                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2296                         // And pal tests
2297                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2298                     }
2299                     else {
2300                         // Corefx stress testing
2301                         assert os == 'Ubuntu'
2302                         assert architecture == 'x64'
2303                         assert lowerConfiguration == 'checked'
2304                         assert isJitStressScenario(scenario)
2305
2306                         // Build coreclr
2307                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2308
2309                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2310
2311                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2312                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2313                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2314                         buildCommands += envScriptCmds
2315
2316                         // Build and text corefx
2317                         def workspaceRelativeFxRoot = "_/fx"
2318                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2319
2320                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2321
2322                         // Archive and process (only) the test results
2323                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2324                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2325                     }
2326                     break
2327                 case 'arm64':
2328                     if (!doCoreFxTesting) {
2329                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2330
2331                         // HACK -- Arm64 does not have corefx jobs yet.
2332                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2333                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2334                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2335                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2336
2337                         // Basic archiving of the build
2338                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2339                     }
2340                     break
2341                 case 'armem':
2342                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2343                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2344
2345                     // default values for Ubuntu
2346                     def arm_abi = "arm"
2347                     def linuxCodeName = "trusty"
2348                     if (os == 'Ubuntu16.04') {
2349                         linuxCodeName = "xenial"
2350                     }
2351                     else if (os == 'Tizen') {
2352                         arm_abi = "armel"
2353                         linuxCodeName = "tizen"
2354                     }
2355
2356                     // Unzip the Windows test binaries first. Exit with 0
2357                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2358
2359                     // Unpack the corefx binaries
2360                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2361                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2362                     if (os != 'Tizen') {
2363                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2364                     }
2365                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2366                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2367                     if (os == 'Ubuntu') {
2368                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2369                     }
2370
2371                     // Call the ARM CI script to cross build and test using docker
2372                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2373                     --mode=docker \\
2374                     --${arm_abi} \\
2375                     --linuxCodeName=${linuxCodeName} \\
2376                     --buildConfig=${lowerConfiguration} \\
2377                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2378                     --coreFxBinDir=./bin/CoreFxBinDir \\
2379                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2380
2381                     // Basic archiving of the build, no pal tests
2382                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2383                     break
2384                 case 'arm':
2385                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2386                     assert (os == 'Ubuntu')
2387
2388                     // Add some useful information to the log file. Ignore return codes.
2389                     buildCommands += "uname -a || true"
2390
2391                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2392                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2393
2394                     def dockerImage = getDockerImageName(architecture, os, true)
2395                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm ${dockerImage} "
2396
2397                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross"
2398
2399                     // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2400                     // download the appropriate CoreFX packages.
2401                     // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2402                     // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2403
2404                     buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2405
2406                     // ZIP up for the test job (created in the flow job code):
2407                     // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2408                     //     used by runtest.sh as the "--coreOverlayDir" argument.
2409                     // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2410                     //     used by runtest.sh as the "--testNativeBinDir" argument.
2411
2412                     // These commands are assumed to be run from the root of the workspace.
2413                     buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root"
2414                     buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests"
2415
2416                     Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2417                     break
2418                 default:
2419                     println("Unknown architecture: ${architecture}");
2420                     assert false
2421                     break
2422             }
2423             break
2424         // editor brace matching: }
2425         default:
2426             println("Unknown os: ${os}");
2427             assert false
2428             break
2429     } // os
2430
2431     return buildCommands
2432 }
2433
2434 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2435 // Returns true if the job should be generated.
2436 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2437 {
2438     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2439     // All other scenarios do Pri-1 testing.
2440     if (scenario == 'innerloop' && !isPR) {
2441         return false
2442     }
2443
2444     // Tizen is only supported for armem architecture
2445     if (os == 'Tizen' && architecture != 'armem') {
2446         return false
2447     }
2448
2449     // Filter based on architecture.
2450
2451     switch (architecture) {
2452         case 'arm64':
2453         case 'arm':
2454             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2455                 return false
2456             }
2457             break
2458         case 'armem':
2459             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2460                 return false
2461             }
2462             break
2463         case 'armlb':
2464             // Do not create armlb jobs
2465             return false
2466         case 'x86_arm_altjit':
2467         case 'x64_arm64_altjit':
2468             if (os != 'Windows_NT') {
2469                 return false
2470             }
2471             break
2472         case 'x86':
2473             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2474                 return false
2475             }
2476             break
2477         case 'x64':
2478             // Everything implemented
2479             break
2480         default:
2481             println("Unknown architecture: ${architecture}")
2482             assert false
2483             break
2484     }
2485
2486     // Which (Windows) build only jobs are required?
2487
2488     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2489
2490     if (isBuildOnly) {
2491         switch (architecture) {
2492             case 'arm':
2493                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2494                 if (!isCoreFxScenario(scenario)) {
2495                     return false
2496                 }
2497                 break
2498             case 'x64':
2499             case 'x86':
2500                 if (!isNormalOrInnerloop) {
2501                     return false
2502                 }
2503                 break
2504             default:
2505                 return false
2506         }
2507     }
2508
2509     // Filter based on scenario.
2510
2511     if (isJitStressScenario(scenario)) {
2512         if (configuration != 'Checked') {
2513             return false
2514         }
2515
2516         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2517         if (!isEnabledOS) {
2518             return false
2519         }
2520
2521         switch (architecture) {
2522             case 'x64':
2523             case 'x86_arm_altjit':
2524             case 'x64_arm64_altjit':
2525                 break
2526
2527             case 'x86':
2528                 // x86 ubuntu: no stress modes
2529                 if (os == 'Ubuntu') {
2530                     return false
2531                 }
2532                 break
2533
2534             case 'arm':
2535                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2536                 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2537                     return false
2538                 }
2539                 break
2540
2541             default:
2542                 // arm64, armlb: stress is handled through flow jobs.
2543                 // armem: no stress jobs for ARM emulator.
2544                 return false
2545         }
2546     }
2547     else if (isR2RScenario(scenario)) {
2548         if (os != 'Windows_NT') {
2549             return false
2550         }
2551         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2552         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2553             return false
2554         }
2555     }
2556     else {
2557         // Skip scenarios
2558         switch (scenario) {
2559             case 'ilrt':
2560                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2561                 // the default scenario build
2562                 if (os != 'Windows_NT') {
2563                     return false
2564                 }
2565                 // Only x64 for now
2566                 if (architecture != 'x64') {
2567                     return false
2568                 }
2569                 // Release only
2570                 if (configuration != 'Release') {
2571                     return false
2572                 }
2573                 break
2574             case 'jitdiff':
2575                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2576                     return false
2577                 }
2578                 if (architecture != 'x64') {
2579                     return false
2580                 }
2581                 if (configuration != 'Checked') {
2582                     return false
2583                 }
2584                 break
2585             case 'longgc':
2586             case 'gcsimulator':
2587                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2588                     return false
2589                 }
2590                 if (architecture != 'x64') {
2591                     return false
2592                 }
2593                 if (configuration != 'Release') {
2594                     return false
2595                 }
2596                 break
2597             case 'gc_reliability_framework':
2598             case 'standalone_gc':
2599                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2600                     return false
2601                 }
2602
2603                 if (architecture != 'x64') {
2604                     return false
2605                 }
2606
2607                 if (configuration != 'Release' && configuration != 'Checked') {
2608                     return false
2609                 }
2610                 break
2611             // We only run Windows and Ubuntu x64 Checked for formatting right now
2612             case 'formatting':
2613                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2614                     return false
2615                 }
2616                 if (architecture != 'x64') {
2617                     return false
2618                 }
2619                 if (configuration != 'Checked') {
2620                     return false
2621                 }
2622                 break
2623             case 'illink':
2624                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2625                     return false
2626                 }
2627                 if (architecture != 'x64' && architecture != 'x86') {
2628                     return false
2629                 }
2630                 break
2631             case 'normal':
2632                 // Nothing skipped
2633                 break
2634             case 'innerloop':
2635                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2636                     return false
2637                 }
2638                 break
2639             default:
2640                 println("Unknown scenario: ${scenario}")
2641                 assert false
2642                 break
2643         }
2644     }
2645
2646     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2647     // runtime behavior, not compile-time behavior, are not interesting.
2648     switch (architecture) {
2649         case 'x86_arm_altjit':
2650         case 'x64_arm64_altjit':
2651             if (isGCStressRelatedTesting(scenario)) {
2652                 return false
2653             }
2654             break
2655         default:
2656             break
2657     }
2658
2659     // The job was not filtered out, so we should generate it!
2660     return true
2661 }
2662
2663 Constants.allScenarios.each { scenario ->
2664     [true, false].each { isPR ->
2665         Constants.architectureList.each { architecture ->
2666             Constants.configurationList.each { configuration ->
2667                 Constants.osList.each { os ->
2668                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2669                     // and reset the os to Windows_NT
2670                     def isBuildOnly = false
2671                     if (os == 'Windows_NT_BuildOnly') {
2672                         isBuildOnly = true
2673                         os = 'Windows_NT'
2674                     }
2675
2676                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2677                         return
2678                     }
2679
2680                     // Calculate names
2681                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2682                     def folderName = getJobFolder(scenario)
2683
2684                     // Create the new job
2685                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2686                     addToViews(newJob, isPR, architecture, os)
2687
2688                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2689
2690                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2691                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2692                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2693
2694                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2695                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2696                     // artifacts produced.
2697                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2698                         // Define the Windows Tests and Corefx build job names
2699                         def lowerConfiguration = configuration.toLowerCase()
2700                         def WindowsTestsName = projectFolder + '/' +
2701                                                Utilities.getFullJobName(project,
2702                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2703                                                                         false)
2704                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2705                                            Utilities.getFolderName(branch)
2706
2707                         def arm_abi = 'arm'
2708                         def corefx_os = 'linux'
2709                         if (os == 'Tizen') {
2710                             arm_abi = 'armel'
2711                             corefx_os = 'tizen'
2712                         }
2713
2714                         // Let's use release CoreFX to test checked CoreCLR,
2715                         // because we do not generate checked CoreFX in CoreFX CI yet.
2716                         def corefx_lowerConfiguration = lowerConfiguration
2717                         if (lowerConfiguration == 'checked') {
2718                             corefx_lowerConfiguration = 'release'
2719                         }
2720
2721                         // Copy the Windows test binaries and the Corefx build binaries
2722                         newJob.with {
2723                             steps {
2724                                 copyArtifacts(WindowsTestsName) {
2725                                     includePatterns('bin/tests/tests.zip')
2726                                     buildSelector {
2727                                         latestSuccessful(true)
2728                                     }
2729                                 }
2730                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2731                                     includePatterns('bin/build.tar.gz')
2732                                     buildSelector {
2733                                         latestSuccessful(true)
2734                                     }
2735                                 }
2736                             } // steps
2737                         } // newJob.with
2738                     }
2739
2740                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2741
2742                     newJob.with {
2743                         steps {
2744                             if (os == 'Windows_NT') {
2745                                 buildCommands.each { buildCommand ->
2746                                     batchFile(buildCommand)
2747                                 }
2748                             }
2749                             else {
2750                                 buildCommands.each { buildCommand ->
2751                                     shell(buildCommand)
2752                                 }
2753                             }
2754                         } // steps
2755                     } // newJob.with
2756
2757                 } // os
2758             } // configuration
2759         } // architecture
2760     } // isPR
2761 } // scenario
2762
2763 // Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job.
2764 // Returns the newly created job.
2765 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2766 {
2767     def osGroup = getOSGroup(os)
2768     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2769
2770     def jobFolder = getJobFolder(scenario)
2771     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2772         parameters {
2773             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2774         }
2775
2776         steps {
2777             // Set up the copies
2778
2779             // Coreclr build we are trying to test
2780             //
2781             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2782
2783             copyArtifacts(inputCoreCLRBuildName) {
2784                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2785                 buildSelector {
2786                     buildNumber('${CORECLR_BUILD}')
2787                 }
2788             }
2789
2790             if (isCoreFxScenario(scenario)) {
2791
2792                 // Only arm supported for corefx testing now.
2793                 assert architecture == 'arm'
2794
2795                 // Unzip CoreFx runtime
2796                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')")
2797
2798                 // Unzip CoreFx tests.
2799                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')")
2800
2801                 // Add the script to run the corefx tests
2802                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2803                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2804                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2805                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2806
2807             } else { // !isCoreFxScenario(scenario)
2808
2809                 // Unzip tests.
2810                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2811
2812                 def buildCommands = ""
2813
2814                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2815                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2816                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2817
2818                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2819                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2820
2821                 // For all jobs
2822                 addEnvVariable("CORE_ROOT", coreRootLocation)
2823                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2824                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2825
2826                 // ARM legacy backend; this is an altjit.
2827                 if (architecture == 'armlb') {
2828                     addEnvVariable("COMPlus_AltJit", "*")
2829                     addEnvVariable("COMPlus_AltJitNgen", "*")
2830                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2831                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2832                 }
2833
2834                 // If we are running a stress mode, we'll set those variables as well
2835                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2836                     def stressValues = null
2837                     if (isJitStressScenario(scenario)) {
2838                         stressValues = Constants.jitStressModeScenarios[scenario]
2839                     }
2840                     else {
2841                         stressValues = Constants.r2rStressScenarios[scenario]
2842                     }
2843
2844                     stressValues.each { key, value ->
2845                         addEnvVariable(key, value)
2846                     }
2847                 }
2848
2849                 if (isR2RScenario(scenario)) {
2850                     // Crossgen the framework assemblies.
2851                     buildCommands += """
2852 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2853 @goto skip_PrecompileAssembly
2854
2855 :PrecompileAssembly
2856 @REM Skip mscorlib since it is already precompiled.
2857 @if /I "%3" == "mscorlib.dll" exit /b 0
2858 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2859
2860 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2861 @if "%errorlevel%" == "-2146230517" (
2862     echo %2 is not a managed assembly.
2863 ) else if "%errorlevel%" == "-2146234344" (
2864     echo %2 is not a managed assembly.
2865 ) else if %errorlevel% neq 0 (
2866     echo Unable to precompile %2
2867 ) else (
2868     echo Precompiled %2
2869 )
2870 @exit /b 0
2871
2872 :skip_PrecompileAssembly
2873 """
2874
2875                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2876                     // crossgen on tests before running them.
2877                     addEnvVariable("RunCrossGen", "true")
2878                 } // isR2RScenario(scenario)
2879
2880                 // Create the smarty command
2881                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2882                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2883                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2884                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2885
2886                 if (architecture == 'armlb') {
2887                     addExclude("LEGACYJIT_FAIL")
2888                 }
2889
2890                 // Exclude tests based on scenario.
2891                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
2892                     addArchSpecificExclude(architecture, excludeTag)
2893                 }
2894
2895                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
2896                 if (scenario == 'innerloop') {
2897                     addExclude("pri1")
2898                 }
2899
2900                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2901                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2902                 addExclude("LONG_RUNNING")
2903
2904                 smartyCommand += "/lstFile Tests.lst"
2905
2906                 def testListArch = [
2907                     'arm64': 'arm64',
2908                     'arm': 'arm',
2909                     'armlb': 'arm'
2910                 ]
2911
2912                 def archLocation = testListArch[architecture]
2913
2914                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2915                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2916                 addCommand("${smartyCommand}")
2917
2918                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2919                 // However, we also need to remove all the variables that were set during this batch file, so we
2920                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2921                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2922                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2923                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2924                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2925                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2926
2927                 // ZIP up the smarty output, no matter what the smarty result.
2928                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2929
2930                 addCommand("echo %errorlevel%")
2931                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2932
2933                 // Use the smarty errorlevel as the script errorlevel.
2934                 addCommand("exit /b %__save_smarty_errorlevel%")
2935
2936                 batchFile(buildCommands)
2937             } // non-corefx testing
2938         } // steps
2939     } // job
2940
2941     if (!isCoreFxScenario(scenario)) {
2942         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2943
2944         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2945         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2946         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2947         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2948     }
2949
2950     return newJob
2951 }
2952
2953 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
2954 // E.g., non-Windows tests.
2955 // Returns the newly created job.
2956 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
2957 {
2958     def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator)
2959
2960     def osGroup = getOSGroup(os)
2961     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2962
2963     def testOpts = ''
2964     def useServerGC = false
2965
2966     // Enable Server GC for Ubuntu PR builds
2967     // REVIEW: why? Does this apply to all architectures? Why only PR?
2968     if (os == 'Ubuntu' && isPR) {
2969         testOpts += ' --useServerGC'
2970         useServerGC = true
2971     }
2972
2973     if (isR2RScenario(scenario)) {
2974
2975         testOpts += ' --crossgen --runcrossgentests'
2976
2977         if (scenario == 'r2r_jitstress1') {
2978             testOpts += ' --jitstress=1'
2979         }
2980         else if (scenario == 'r2r_jitstress2') {
2981             testOpts += ' --jitstress=2'
2982         }
2983         else if (scenario == 'r2r_jitstressregs1') {
2984             testOpts += ' --jitstressregs=1'
2985         }
2986         else if (scenario == 'r2r_jitstressregs2') {
2987             testOpts += ' --jitstressregs=2'
2988         }
2989         else if (scenario == 'r2r_jitstressregs3') {
2990             testOpts += ' --jitstressregs=3'
2991         }
2992         else if (scenario == 'r2r_jitstressregs4') {
2993             testOpts += ' --jitstressregs=4'
2994         }
2995         else if (scenario == 'r2r_jitstressregs8') {
2996             testOpts += ' --jitstressregs=8'
2997         }
2998         else if (scenario == 'r2r_jitstressregs0x10') {
2999             testOpts += ' --jitstressregs=0x10'
3000         }
3001         else if (scenario == 'r2r_jitstressregs0x80') {
3002             testOpts += ' --jitstressregs=0x80'
3003         }
3004         else if (scenario == 'r2r_jitstressregs0x1000') {
3005             testOpts += ' --jitstressregs=0x1000'
3006         }
3007         else if (scenario == 'r2r_jitminopts') {
3008             testOpts += ' --jitminopts'
3009         }
3010         else if (scenario == 'r2r_jitforcerelocs') {
3011             testOpts += ' --jitforcerelocs'
3012         }
3013         else if (scenario == 'r2r_gcstress15') {
3014             testOpts += ' --gcstresslevel=0xF'
3015         }
3016     }
3017     else if (scenario == 'jitdiff') {
3018         testOpts += ' --jitdisasm --crossgen'
3019     }
3020     else if (scenario == 'illink') {
3021         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3022     }
3023     else if (isLongGc(scenario)) {
3024         // Long GC tests behave very poorly when they are not
3025         // the only test running (many of them allocate until OOM).
3026         testOpts += ' --sequential'
3027
3028         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3029         // for running long GC and GCSimulator tests, respectively. We don't use them
3030         // here because using a playlist file produces much more readable output on the CI machines
3031         // and reduces running time.
3032         //
3033         // The Long GC playlist contains all of the tests that are
3034         // going to be run. The GCSimulator playlist contains all of
3035         // the GC simulator tests.
3036         if (scenario == 'longgc') {
3037             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3038         }
3039         else if (scenario == 'gcsimulator') {
3040             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3041         }
3042     }
3043     else if (isGcReliabilityFramework(scenario)) {
3044         testOpts += ' --build-overlay-only'
3045     }
3046     else if (scenario == 'standalone_gc') {
3047         if (osGroup == 'OSX') {
3048             testOpts += ' --gcname=libclrgc.dylib'
3049         }
3050         else if (osGroup == 'Linux') {
3051             testOpts += ' --gcname=libclrgc.so'
3052         }
3053         else {
3054             println("Unexpected OS group: ${osGroup} for os ${os}")
3055             assert false
3056         }
3057     }
3058
3059     def jobFolder = getJobFolder(scenario)
3060     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3061         parameters {
3062             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3063             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3064         }
3065
3066         steps {
3067             // Set up the copies
3068
3069             // Coreclr build containing the tests and mscorlib
3070             // pri1 jobs still need to copy windows_nt built tests
3071             assert inputTestsBuildName != null
3072             copyArtifacts(inputTestsBuildName) {
3073                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3074                 buildSelector {
3075                     buildNumber('${CORECLR_WINDOWS_BUILD}')
3076                 }
3077             }
3078
3079             // Coreclr build we are trying to test
3080             //
3081             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3082
3083             copyArtifacts(inputCoreCLRBuildName) {
3084                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3085                 buildSelector {
3086                     buildNumber('${CORECLR_BUILD}')
3087                 }
3088             }
3089
3090             if (isUbuntuArmJob) {
3091                 // Add some useful information to the log file. Ignore return codes.
3092                 shell("uname -a || true")
3093             }
3094
3095             if (architecture == 'arm64') {
3096                 shell("mkdir -p ./bin/CoreFxBinDir")
3097                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
3098                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
3099             }
3100             else if (architecture == 'x86') {
3101                 shell("mkdir ./bin/CoreFxNative")
3102
3103                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
3104
3105                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3106                     includePatterns('bin/build.tar.gz')
3107                     targetDirectory('bin/CoreFxNative')
3108                     buildSelector {
3109                         latestSuccessful(true)
3110                     }
3111                 }
3112
3113                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3114             }
3115
3116             // Unzip the tests first.  Exit with 0
3117             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3118             shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3119
3120             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3121             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3122             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3123             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3124             // copied correctly.
3125             if (isUbuntuArmJob) {
3126                 def lowerConfiguration = configuration.toLowerCase()
3127                 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root
3128                 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests
3129             }
3130             else {
3131                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3132             }
3133
3134             // Execute the tests
3135             def runDocker = isNeedDocker(architecture, os, false)
3136             def dockerPrefix = ""
3137             def dockerCmd = ""
3138             if (runDocker) {
3139                 def dockerImage = getDockerImageName(architecture, os, false)
3140                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3141                 dockerCmd = dockerPrefix + "${dockerImage} "
3142             }
3143
3144             // If we are running a stress mode, we'll set those variables first
3145             if (isJitStressScenario(scenario)) {
3146                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3147                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3148                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3149                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3150                 shell("${envScriptCmds}")
3151                 testOpts += " --test-env=${scriptFileName}"
3152             }
3153
3154             // TODO: how to handle GCStress-related testing for Ubuntu/arm?
3155             if (isGCStressRelatedTesting(scenario)) {
3156                 shell('./init-tools.sh')
3157             }
3158
3159             def runScript = ""
3160             if (isUbuntuArmJob) {
3161                 // Use 'runtesttilstable.sh' to rerun failing tests (in sequential mode);
3162                 // there are many tests that pass on rerun (currently), and we don't want
3163                 // that flakiness to affect overall test job robustness.
3164                 runScript = "${dockerCmd}./tests/runtesttilstable.sh"
3165             } else {
3166                 runScript = "${dockerCmd}./tests/runtest.sh"
3167             }
3168
3169             shell("""\
3170 ${runScript} \\
3171     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3172     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3173     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3174     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3175
3176             if (isGcReliabilityFramework(scenario)) {
3177                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3178                 if (useServerGC) {
3179                     if (runDocker) {
3180                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3181                     }
3182                     else {
3183                         shell("export COMPlus_gcServer=1")
3184                     }
3185                 }
3186
3187                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3188             }
3189         } // steps
3190     } // job
3191
3192     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3193     if (os in ['Ubuntu']) {
3194         SummaryBuilder summaries = new SummaryBuilder()
3195         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3196         summaries.emit(newJob)
3197     }
3198
3199     Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3200     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3201
3202     return newJob
3203 }
3204
3205 // Create a test job that will be used by a flow job.
3206 // Returns the newly created job.
3207 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3208 {
3209     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3210
3211     def newJob = null
3212     if (windowsArmJob) {
3213         assert inputTestsBuildName == null
3214         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3215     } else {
3216         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3217     }
3218
3219     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3220
3221     addToViews(newJob, isPR, architecture, os)
3222
3223     if (scenario == 'jitdiff') {
3224         def osGroup = getOSGroup(os)
3225         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3226     }
3227
3228     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3229     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3230
3231     return newJob
3232 }
3233
3234 // Create a flow job to tie together a build job with the given test job.
3235 // Returns the new flow job.
3236 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3237 {
3238     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3239     // Linux CoreCLR test
3240     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3241     def jobFolder = getJobFolder(scenario)
3242
3243     def newFlowJob = null
3244
3245     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3246     if (windowsArmJob) {
3247
3248         assert inputTestsBuildName == null
3249
3250         // For Windows arm jobs there is no reason to build a parallel test job.
3251         // The product build supports building and archiving the tests.
3252
3253         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3254                         buildFlow("""\
3255 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3256
3257 // And then build the test build
3258 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3259 """)
3260         }
3261         JobReport.Report.addReference(inputCoreCLRBuildName)
3262         JobReport.Report.addReference(fullTestJobName)
3263     }
3264     else {
3265         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3266                         buildFlow("""\
3267 // Build the input jobs in parallel
3268 parallel (
3269 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3270 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3271 )
3272
3273 // And then build the test build
3274 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3275                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3276 """)
3277         }
3278         JobReport.Report.addReference(inputCoreCLRBuildName)
3279         JobReport.Report.addReference(inputTestsBuildName)
3280         JobReport.Report.addReference(fullTestJobName)
3281     }
3282
3283     addToViews(newFlowJob, isPR, architecture, os)
3284
3285     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3286
3287     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3288     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3289
3290     return newFlowJob
3291 }
3292
3293 // Determine if we should generate a flow job for the given parameters.
3294 // Returns true if the job should be generated.
3295 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3296 {
3297     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3298     // All other scenarios do Pri-1 testing.
3299     if (scenario == 'innerloop' && !isPR) {
3300         return false
3301     }
3302
3303     // Filter based on OS and architecture.
3304
3305     switch (architecture) {
3306         case 'arm64':
3307             if (os != "Ubuntu" && os != "Windows_NT") {
3308                 return false
3309             }
3310             break
3311         case 'armlb':
3312             if (os != 'Windows_NT') {
3313                 return false
3314             }
3315             // Do not create armlb windows jobs.
3316             return false
3317         case 'arm':
3318             if (os != "Ubuntu" && os != "Windows_NT") {
3319                 return false
3320             }
3321             break
3322         case 'x86':
3323             if (os != "Ubuntu") {
3324                 return false
3325             }
3326             break
3327         case 'x64':
3328             if (!(os in Constants.crossList)) {
3329                 return false
3330             }
3331             if (os == "Windows_NT") {
3332                 return false
3333             }
3334             break
3335         case 'armem':
3336         case 'x86_arm_altjit':
3337         case 'x64_arm64_altjit':
3338             // No flow jobs
3339             return false
3340         default:
3341             println("Unknown architecture: ${architecture}")
3342             assert false
3343             break
3344     }
3345
3346     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3347
3348     // Filter based on scenario in OS.
3349
3350     if (os == 'Windows_NT') {
3351         if (!isArmWindowsScenario(scenario)) {
3352             return false
3353         }
3354     }
3355     else {
3356         // Non-Windows
3357         if (architecture == 'arm64') {
3358             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3359                 return false
3360             }
3361         }
3362         else if (architecture == 'arm') {
3363             if (!(scenario in Constants.validLinuxArmScenarios)) {
3364                 return false
3365             }
3366         }
3367         else if (architecture == 'x86') {
3368             // Linux/x86 only want innerloop and default test
3369             if (!isNormalOrInnerloop) {
3370                 return false
3371             }
3372         }
3373     }
3374
3375     // For CentOS, we only want Checked/Release builds.
3376     if (os == 'CentOS7.1') {
3377         if (configuration != 'Checked' && configuration != 'Release') {
3378             return false
3379         }
3380         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3381             return false
3382         }
3383     }
3384
3385     // For RedHat and Debian, we only do Release builds.
3386     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3387         if (configuration != 'Release') {
3388             return false
3389         }
3390         if (!isNormalOrInnerloop) {
3391             return false
3392         }
3393     }
3394
3395     // Next, filter based on scenario.
3396
3397     if (isJitStressScenario(scenario)) {
3398         if (configuration != 'Checked') {
3399             return false
3400         }
3401
3402         // CoreFx JIT stress tests currently only implemented for Windows ARM.
3403         if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) {
3404             return false
3405         }
3406     }
3407     else if (isR2RBaselineScenario(scenario)) {
3408         if (configuration != 'Checked' && configuration != 'Release') {
3409             return false
3410         }
3411     }
3412     else if (isR2RStressScenario(scenario)) {
3413         if (configuration != 'Checked') {
3414             return false
3415         }
3416     }
3417     else {
3418         // Skip scenarios
3419         switch (scenario) {
3420             case 'ilrt':
3421             case 'longgc':
3422             case 'gcsimulator':
3423                 // Long GC tests take a long time on non-Release builds
3424                 // ilrt is also Release only
3425                 if (configuration != 'Release') {
3426                     return false
3427                 }
3428                 break
3429
3430             case 'jitdiff':
3431                 if (configuration != 'Checked') {
3432                     return false
3433                 }
3434                 break
3435
3436             case 'gc_reliability_framework':
3437             case 'standalone_gc':
3438                 if (configuration != 'Release' && configuration != 'Checked') {
3439                     return false
3440                 }
3441                 break
3442
3443             case 'formatting':
3444                 return false
3445             case 'illink':
3446                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3447                     return false
3448                 }
3449                 break
3450
3451             case 'normal':
3452                 // Nothing skipped
3453                 break
3454
3455             case 'innerloop':
3456                 // Nothing skipped
3457                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3458                     return false
3459                 }
3460                 break
3461
3462             default:
3463                 println("Unknown scenario: ${scenario}")
3464                 assert false
3465                 break
3466         }
3467     }
3468
3469     // The job was not filtered out, so we should generate it!
3470     return true
3471 }
3472
3473 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows.
3474 // Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it.
3475 Constants.allScenarios.each { scenario ->
3476     [true, false].each { isPR ->
3477         Constants.architectureList.each { architecture ->
3478             Constants.configurationList.each { configuration ->
3479                 Constants.osList.each { os ->
3480
3481                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3482                         return
3483                     }
3484
3485                     // Figure out the job name of the CoreCLR build the test will depend on.
3486
3487                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3488                     def inputCoreCLRBuildIsBuildOnly = false
3489                     if (isCoreFxScenario(scenario)) {
3490                         // Every CoreFx test depends on its own unique build.
3491                         inputCoreCLRBuildScenario = scenario
3492                         inputCoreCLRBuildIsBuildOnly = true
3493                     }
3494                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3495                     def inputCoreCLRBuildName = projectFolder + '/' +
3496                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3497
3498                     // Figure out the name of the build job that the test job will depend on.
3499                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3500                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3501
3502                     def inputTestsBuildName = null
3503
3504                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3505                     if (!windowsArmJob) {
3506                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3507
3508                         def inputTestsBuildArch = architecture
3509                         if (architecture == "arm64") {
3510                             // Use the x64 test build for arm64 unix
3511                             inputTestsBuildArch = "x64"
3512                         }
3513                         else if (architecture == "arm") {
3514                             // Use the x86 test build for arm unix
3515                             inputTestsBuildArch = "x86"
3516                         }
3517
3518                         def inputTestsBuildIsBuildOnly = true
3519
3520                         inputTestsBuildName = projectFolder + '/' +
3521                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3522                     }
3523
3524                     // =============================================================================================
3525                     // Create the test job
3526                     // =============================================================================================
3527
3528                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3529
3530                     // =============================================================================================
3531                     // Create a build flow to join together the build and tests required to run this test.
3532                     // =============================================================================================
3533
3534                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3535                         // Do not create the flow job for RHEL jobs.
3536                         return
3537                     }
3538
3539                     def fullTestJobName = projectFolder + '/' + testJob.name
3540                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3541
3542                 } // os
3543             } // configuration
3544         } // architecture
3545     } // isPR
3546 } // scenario
3547
3548 JobReport.Report.generateJobReport(out)
3549
3550 // Make the call to generate the help job
3551 Utilities.createHelperJob(this, project, branch,
3552     "Welcome to the ${project} Repository",  // This is prepended to the help message
3553     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3554
3555 Utilities.addCROSSCheck(this, project, branch)