netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'corefx_baseline'
 266                // 'corefx_minopts'
 267                // 'corefx_tieredcompilation'
 268                // 'corefx_jitstress1'
 269                // 'corefx_jitstress2'
 270                // 'corefx_jitstressregs1'
 271                // 'corefx_jitstressregs2'
 272                // 'corefx_jitstressregs3'
 273                // 'corefx_jitstressregs4'
 274                // 'corefx_jitstressregs8'
 275                // 'corefx_jitstressregs0x10'
 276                // 'corefx_jitstressregs0x80'
 277                // 'corefx_jitstressregs0x1000'
 278                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 279                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 280                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 281                'heapverify1':                            [],
 282                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 283                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 284                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 285                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 288
 289                //
 290                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 291                //
 292
 293                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 294                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 312     ]
 313
 314     def static validLinuxArm64Scenarios = [
 315         'normal',
 316         'r2r',
 317         'innerloop',
 318         'gcstress0x3',
 319         'gcstress0xc'
 320     ]
 321
 322     def static configurationList = ['Debug', 'Checked', 'Release']
 323
 324     // This is the set of architectures
 325     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 326 }
 327
 328 // **************************************************************
 329 // Create some specific views
 330 //
 331 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 332 // views based on a single regular expression. These views will be generated by adding a
 333 // specific set of jobs to them.
 334 //
 335 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 336 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 337 // those are required, don't add them (which simplifies the view pages, as well).
 338 // **************************************************************
 339
 340 class Views {
 341     def static MergeJobView = null
 342     def static PeriodicJobView = null
 343     def static ArchitectureViews = [:]
 344     def static OSViews = [:]
 345 }
 346
 347 // MergeJobView: include all jobs that execute when a PR change is merged.
 348 Views.MergeJobView = listView('Merge') {
 349     recurse()
 350     columns {
 351         status()
 352         weather()
 353         name()
 354         lastSuccess()
 355         lastFailure()
 356         lastDuration()
 357         buildButton()
 358     }
 359 }
 360
 361 // PeriodicJobView: include all jobs that execute on a schedule
 362 Views.PeriodicJobView = listView('Periodic') {
 363     recurse()
 364     columns {
 365         status()
 366         weather()
 367         name()
 368         lastSuccess()
 369         lastFailure()
 370         lastDuration()
 371         buildButton()
 372     }
 373 }
 374
 375 // Create a view for non-PR jobs for each architecture.
 376 Constants.architectureList.each { architecture ->
 377     Views.ArchitectureViews[architecture] = listView(architecture) {
 378         recurse()
 379         columns {
 380             status()
 381             weather()
 382             name()
 383             lastSuccess()
 384             lastFailure()
 385             lastDuration()
 386             buildButton()
 387         }
 388     }
 389 }
 390
 391 // Create a view for non-PR jobs for each OS.
 392 Constants.osList.each { os ->
 393     // Don't create one for the special 'Windows_NT_BuildOnly'
 394     if (os == 'Windows_NT_BuildOnly') {
 395         return
 396     }
 397     Views.OSViews[os] = listView(os) {
 398         recurse()
 399         columns {
 400             status()
 401             weather()
 402             name()
 403             lastSuccess()
 404             lastFailure()
 405             lastDuration()
 406             buildButton()
 407         }
 408     }
 409 }
 410
 411 def static addToMergeView(def job) {
 412     Views.MergeJobView.with {
 413         jobs {
 414             name(job.name)
 415         }
 416     }
 417 }
 418
 419 def static addToPeriodicView(def job) {
 420     Views.PeriodicJobView.with {
 421         jobs {
 422             name(job.name)
 423         }
 424     }
 425 }
 426
 427 def static addToViews(def job, def isPR, def architecture, def os) {
 428     if (isPR) {
 429         // No views want PR jobs currently.
 430         return
 431     }
 432
 433     // Add to architecture view.
 434     Views.ArchitectureViews[architecture].with {
 435         jobs {
 436             name(job.name)
 437         }
 438     }
 439
 440     // Add to OS view.
 441     Views.OSViews[os].with {
 442         jobs {
 443             name(job.name)
 444         }
 445     }
 446 }
 447
 448 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 449     addToPeriodicView(job)
 450     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 451 }
 452
 453 def static addGithubPushTriggerHelper(def job) {
 454     addToMergeView(job)
 455     Utilities.addGithubPushTrigger(job)
 456 }
 457
 458
 459 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 460     assert os instanceof String
 461     assert architecture instanceof String
 462
 463     def armArches = ['arm', 'armlb', 'arm64']
 464     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 465
 466     if (!(architecture in armArches)) {
 467         assert options == null
 468         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 469
 470         return
 471     }
 472
 473     // This is an arm(64) job.
 474     //
 475     // There are several options.
 476     //
 477     // Windows_NT
 478     //
 479     // Arm32 (Build) -> latest-arm64
 480     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 481     // Arm32 (Test)  -> arm64-windows_nt
 482     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 483     //
 484     // Arm64 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 486     // Arm64 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 488     //
 489     // Ubuntu
 490     //
 491     // Arm32 (Build) -> arm-cross-latest
 492     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 493     // Arm32 (Test)  -> NYI Arch not supported
 494     //       |->
 495     //
 496     // Arm64 (Build) -> arm64-cross-latest
 497     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 498     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 499     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 500     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 501     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 502
 503     // This has to be a arm arch
 504     assert architecture in armArches
 505     if (os == "Windows_NT") {
 506         // Arm(64) Windows jobs share the same machines for now
 507         def isBuild = options['use_arm64_build_machine'] == true
 508
 509         if (isBuild == true) {
 510             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 511         } else {
 512             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 513         }
 514     } else {
 515         assert os != 'Windows_NT'
 516         assert os in supportedArmLinuxOs
 517
 518         if (architecture == 'arm' || architecture == 'armlb') {
 519             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 520         } else {
 521             // Arm64 Linux
 522             if (options['is_build_only'] == true) {
 523                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 524             } else {
 525                 // Arm64 Test Machines
 526                 if (options['large_pages'] == false) {
 527                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 528                 } else {
 529                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 530                 }
 531             }
 532         }
 533     }
 534 }
 535
 536 def static isGCStressRelatedTesting(def scenario) {
 537     // The 'r2r_gcstress15' scenario is a basic scenario.
 538     // Detect it and make it a GCStress related.
 539     if (scenario == 'r2r_gcstress15')
 540     {
 541         return true;
 542     }
 543
 544     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 545     def scenarioName = scenario.toLowerCase()
 546     def isGCStressTesting = false
 547     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 548         if (k in gcStressTestEnvVars) {
 549             isGCStressTesting = true;
 550         }
 551     }
 552     return isGCStressTesting
 553 }
 554
 555 def static isCoreFxScenario(def scenario) {
 556     def corefx_prefix = 'corefx_'
 557     if (scenario.length() < corefx_prefix.length()) {
 558         return false
 559     }
 560     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 561 }
 562
 563 def static isR2RBaselineScenario(def scenario) {
 564     return (scenario == 'r2r')
 565 }
 566
 567 def static isR2RStressScenario(def scenario) {
 568     return Constants.r2rStressScenarios.containsKey(scenario)
 569 }
 570
 571 def static isR2RScenario(def scenario) {
 572     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 573 }
 574
 575 def static isJitStressScenario(def scenario) {
 576     return Constants.jitStressModeScenarios.containsKey(scenario)
 577 }
 578
 579 def static isLongGc(def scenario) {
 580     return (scenario == 'longgc' || scenario == 'gcsimulator')
 581 }
 582
 583 def static isJitDiff(def scenario) {
 584     return (scenario == 'jitdiff')
 585 }
 586
 587 def static isGcReliabilityFramework(def scenario) {
 588     return (scenario == 'gc_reliability_framework')
 589 }
 590
 591 def static isArmWindowsScenario(def scenario) {
 592     return Constants.validArmWindowsScenarios.containsKey(scenario)
 593 }
 594
 595 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 596     if (isBuildOnly == true) {
 597         os = 'Windows_NT_buildOnly'
 598     }
 599
 600     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 601
 602     if (validOsPrTriggerArchConfigs == null) {
 603         return false
 604     }
 605
 606     if (validOsPrTriggerArchConfigs[architecture] != null) {
 607         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 608
 609         if (!(configuration in validOsPrTriggerConfigs)) {
 610             return false
 611         }
 612     } else {
 613         return false
 614     }
 615
 616     return true
 617 }
 618
 619 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 620     // 2 hours (120 minutes) is the default timeout
 621     def timeout = 120
 622     def innerLoop = (scenario == "innerloop")
 623
 624     if (!innerLoop) {
 625         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 626         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 627         timeout = 240
 628     }
 629
 630     if (!isBuildOnly) {
 631         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 632         if (isGCStressRelatedTesting(scenario)) {
 633             timeout = 4320
 634         }
 635         else if (isCoreFxScenario(scenario)) {
 636             timeout = 360
 637         }
 638         else if (isJitStressScenario(scenario)) {
 639             timeout = 240
 640         }
 641         else if (isR2RBaselineScenario(scenario)) {
 642             timeout = 240
 643         }
 644         else if (isLongGc(scenario)) {
 645             timeout = 1440
 646         }
 647         else if (isJitDiff(scenario)) {
 648             timeout = 240
 649         }
 650         else if (isGcReliabilityFramework(scenario)) {
 651             timeout = 1440
 652         }
 653         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 654             timeout = 240
 655         }
 656     }
 657
 658     if (configuration == 'Debug') {
 659         // Debug runs can be very slow. Add an hour.
 660         timeout += 60
 661     }
 662
 663     // If we've changed the timeout from the default, set it in the job.
 664
 665     if (timeout != 120) {
 666         Utilities.setJobTimeout(newJob, timeout)
 667     }
 668 }
 669
 670 def static getJobFolder(def scenario) {
 671     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 672         return 'jitstress'
 673     }
 674     if (scenario == 'illink') {
 675         return 'illink'
 676     }
 677     return ''
 678 }
 679
 680 def static getStressModeDisplayName(def scenario) {
 681     def displayStr = ''
 682     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 683         def prefixLength = 'COMPlus_'.length()
 684         if (k.length() >= prefixLength) {
 685             def modeName = k.substring(prefixLength, k.length())
 686             displayStr += ' ' + modeName + '=' + v
 687         }
 688     }
 689
 690     if (isCoreFxScenario(scenario)) {
 691         displayStr = ('CoreFx ' + displayStr).trim()
 692     }
 693
 694     return displayStr
 695 }
 696
 697 def static getR2RDisplayName(def scenario) {
 698     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 699     def displayStr = scenario
 700     def prefixLength = 'r2r_'.length()
 701     if (displayStr.length() >= prefixLength) {
 702         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 703     } else if (scenario == 'r2r') {
 704         displayStr = "R2R"
 705     }
 706     return displayStr
 707 }
 708
 709 //
 710 // Functions to create an environment script.
 711 //      envScriptCreate -- initialize the script (call first)
 712 //      envScriptFinalize -- finalize the script (call last)
 713 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 714 //      envScriptAppendExistingScript -- append an existing script to the generated script
 715 //
 716 // Each script returns a string of commands. Concatenate all the strings together before
 717 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 718 //
 719
 720 // Initialize the environment setting script.
 721 def static envScriptCreate(def os, def stepScriptLocation) {
 722     def stepScript = ''
 723     if (os == 'Windows_NT') {
 724         stepScript += "echo Creating TestEnv script\r\n"
 725         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 726
 727         // Create at least an empty script.
 728         stepScript += "echo. > ${stepScriptLocation}\r\n"
 729     }
 730     else {
 731         stepScript += "echo Creating environment setting script\n"
 732         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 733     }
 734
 735     return stepScript
 736 }
 737
 738 // Generates the string for setting stress mode variables.
 739 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 740     def stepScript = ''
 741     if (os == 'Windows_NT') {
 742         stressModeVars.each{ k, v ->
 743             // Write out what we are writing to the script file
 744             stepScript += "echo Setting ${k}=${v}\r\n"
 745             // Write out the set itself to the script file`
 746             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 747         }
 748     }
 749     else {
 750         stressModeVars.each{ k, v ->
 751             // Write out what we are writing to the script file
 752             stepScript += "echo Setting ${k}=${v}\n"
 753             // Write out the set itself to the script file`
 754             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 755         }
 756     }
 757
 758     return stepScript
 759 }
 760
 761 // Append an existing script to an environment script.
 762 // Returns string of commands to do this.
 763 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 764     assert (os == 'Windows_NT')
 765     def stepScript = ''
 766
 767     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 768     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 769
 770     return stepScript
 771 }
 772
 773 // Finalize an environment setting script.
 774 // Returns string of commands to do this.
 775 def static envScriptFinalize(def os, def stepScriptLocation) {
 776     def stepScript = ''
 777
 778     if (os == 'Windows_NT') {
 779         // Display the resulting script. This is useful when looking at the output log file.
 780         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 781         stepScript += "type ${stepScriptLocation}\r\n"
 782     }
 783     else {
 784         stepScript += "chmod +x ${stepScriptLocation}\n"
 785     }
 786
 787     return stepScript
 788 }
 789
 790 def static isNeedDocker(def architecture, def os, def isBuild) {
 791     if (isBuild) {
 792         if (architecture == 'x86' && os == 'Ubuntu') {
 793             return true
 794         }
 795         else if (architecture == 'arm') {
 796             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 797                 return true
 798             }
 799         }
 800     }
 801     else {
 802         if (architecture == 'x86' && os == 'Ubuntu') {
 803             return true
 804         }
 805     }
 806     return false
 807 }
 808
 809 def static getDockerImageName(def architecture, def os, def isBuild) {
 810     // We must change some docker private images to official later
 811     if (isBuild) {
 812         if (architecture == 'x86' && os == 'Ubuntu') {
 813             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 814         }
 815         else if (architecture == 'arm') {
 816             if (os == 'Ubuntu') {
 817                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 818             }
 819             else if (os == 'Ubuntu16.04') {
 820                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 821             }
 822             else if (os == 'Tizen') {
 823                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 824             }
 825         }
 826     }
 827     else {
 828         if (architecture == 'x86' && os == 'Ubuntu') {
 829             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 830         }
 831     }
 832     println("Unknown architecture to use docker: ${architecture} ${os}");
 833     assert false
 834 }
 835
 836 // Calculates the name of the build job based on some typical parameters.
 837 //
 838 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 839     // If the architecture is x64, do not add that info into the build name.
 840     // Need to change around some systems and other builds to pick up the right builds
 841     // to do that.
 842
 843     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 844     if (isBuildOnly) {
 845         suffix += '_bld'
 846     }
 847     def baseName = ''
 848     switch (architecture) {
 849         case 'x64':
 850             if (scenario == 'normal') {
 851                 // For now we leave x64 off of the name for compatibility with other jobs
 852                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 853             }
 854             else if (scenario == 'formatting') {
 855                 // we don't care about the configuration for the formatting job. It runs all configs
 856                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 857             }
 858             else {
 859                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 860             }
 861             break
 862         case 'arm64':
 863             if (os.toLowerCase() == "windows_nt") {
 864                 // These are cross builds
 865                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 866             }
 867             else {
 868                 // Defaults to a small page size set of machines.
 869                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 870             }
 871             break
 872         case 'arm':
 873             // These are cross builds
 874             if (os == 'Tizen') {
 875                 // ABI: softfp
 876                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 877             }
 878             else {
 879                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 880             }
 881             break
 882         case 'armlb':
 883             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 884             break
 885         case 'x86':
 886         case 'x86_arm_altjit':
 887         case 'x64_arm64_altjit':
 888             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 889             break
 890         default:
 891             println("Unknown architecture: ${architecture}");
 892             assert false
 893             break
 894     }
 895
 896     return baseName + suffix
 897 }
 898
 899 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 900
 901     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 902     if (os == 'Windows_NT') {
 903         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 904             if (!(branch in Constants.WindowsArm64Branches)) {
 905                 return
 906             }
 907         }
 908     }
 909
 910     // Check scenario.
 911     switch (scenario) {
 912         case 'innerloop':
 913             break
 914         case 'normal':
 915             switch (architecture) {
 916                 case 'x64':
 917                 case 'x86':
 918                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 919                         addPeriodicTriggerHelper(job, '@daily')
 920                     }
 921                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 922                         addGithubPushTriggerHelper(job)
 923                     }
 924                     break
 925                 case 'arm':
 926                 case 'armlb':
 927                 case 'x86_arm_altjit':
 928                 case 'x64_arm64_altjit':
 929                     addGithubPushTriggerHelper(job)
 930                     break
 931                 case 'arm64':
 932                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 933                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 934                     break
 935                 default:
 936                     println("Unknown architecture: ${architecture}");
 937                     assert false
 938                     break
 939             }
 940             break
 941         case 'r2r':
 942             assert !(os in bidailyCrossList)
 943             // r2r gets a push trigger for checked/release
 944             if (configuration == 'Checked' || configuration == 'Release') {
 945                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 946                 if (architecture == 'x64' && os != 'OSX10.12') {
 947                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 948                     if (isFlowJob || os == 'Windows_NT') {
 949                         addGithubPushTriggerHelper(job)
 950                     }
 951                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 952                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 953                     if (isFlowJob) {
 954                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 955                     }
 956                 }
 957                 // For x86, only add per-commit jobs for Windows
 958                 else if (architecture == 'x86') {
 959                     if (os == 'Windows_NT') {
 960                         addGithubPushTriggerHelper(job)
 961                     }
 962                 }
 963                 // arm64 r2r jobs should only run daily.
 964                 else if (architecture == 'arm64') {
 965                     if (os == 'Windows_NT') {
 966                         addPeriodicTriggerHelper(job, '@daily')
 967                     }
 968                 }
 969             }
 970             break
 971         case 'r2r_jitstress1':
 972         case 'r2r_jitstress2':
 973         case 'r2r_jitstressregs1':
 974         case 'r2r_jitstressregs2':
 975         case 'r2r_jitstressregs3':
 976         case 'r2r_jitstressregs4':
 977         case 'r2r_jitstressregs8':
 978         case 'r2r_jitstressregs0x10':
 979         case 'r2r_jitstressregs0x80':
 980         case 'r2r_jitstressregs0x1000':
 981         case 'r2r_jitminopts':
 982         case 'r2r_jitforcerelocs':
 983         case 'r2r_gcstress15':
 984             assert !(os in bidailyCrossList)
 985
 986             // GCStress=C is currently not supported on OS X
 987             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 988                 break
 989             }
 990
 991             // GC Stress 15 r2r gets a push trigger for checked/release
 992             if (configuration == 'Checked' || configuration == 'Release') {
 993                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 994                 if (architecture == 'x64') {
 995                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 996                     if (isFlowJob || os == 'Windows_NT') {
 997                         // Add a weekly periodic trigger
 998                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 999                     }
1000                 }
1001                 // For x86, only add per-commit jobs for Windows
1002                 else if (architecture == 'x86') {
1003                     if (os == 'Windows_NT') {
1004                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1005                     }
1006                 }
1007             }
1008             break
1009         case 'longgc':
1010             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1011             assert configuration == 'Release'
1012             assert architecture == 'x64'
1013             addPeriodicTriggerHelper(job, '@daily')
1014             // TODO: Add once external email sending is available again
1015             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1016             break
1017         case 'gcsimulator':
1018             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1019             assert configuration == 'Release'
1020             assert architecture == 'x64'
1021             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1022             // TODO: Add once external email sending is available again
1023             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1024             break
1025         case 'standalone_gc':
1026             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1027             assert (configuration == 'Release' || configuration == 'Checked')
1028             // TODO: Add once external email sending is available again
1029             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1030             addPeriodicTriggerHelper(job, '@daily')
1031             break
1032         case 'gc_reliability_framework':
1033             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1034             assert (configuration == 'Release' || configuration == 'Checked')
1035             // Only triggered by phrase.
1036             break
1037         case 'ilrt':
1038             assert !(os in bidailyCrossList)
1039             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1040             if (architecture == 'x64' && configuration == 'Release') {
1041                 // We don't expect to see a job generated except in these scenarios
1042                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1043                 if (isFlowJob || os == 'Windows_NT') {
1044                     addPeriodicTriggerHelper(job, '@daily')
1045                 }
1046             }
1047             break
1048         case 'jitdiff':
1049             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1050             assert configuration == 'Checked'
1051             assert (architecture == 'x64' || architecture == 'x86')
1052             addGithubPushTriggerHelper(job)
1053             break
1054         case 'formatting':
1055             assert (os == 'Windows_NT' || os == "Ubuntu")
1056             assert architecture == 'x64'
1057             addGithubPushTriggerHelper(job)
1058             break
1059         case 'jitstressregs1':
1060         case 'jitstressregs2':
1061         case 'jitstressregs3':
1062         case 'jitstressregs4':
1063         case 'jitstressregs8':
1064         case 'jitstressregs0x10':
1065         case 'jitstressregs0x80':
1066         case 'jitstressregs0x1000':
1067         case 'minopts':
1068         case 'forcerelocs':
1069         case 'jitstress1':
1070         case 'jitstress2':
1071         case 'jitstress2_jitstressregs1':
1072         case 'jitstress2_jitstressregs2':
1073         case 'jitstress2_jitstressregs3':
1074         case 'jitstress2_jitstressregs4':
1075         case 'jitstress2_jitstressregs8':
1076         case 'jitstress2_jitstressregs0x10':
1077         case 'jitstress2_jitstressregs0x80':
1078         case 'jitstress2_jitstressregs0x1000':
1079         case 'tailcallstress':
1080         case 'jitsse2only':
1081         case 'jitnosimd':
1082         case 'jitnox86hwintrinsic':
1083         case 'jitincompletehwintrinsic':
1084         case 'jitx86hwintrinsicnoavx':
1085         case 'jitx86hwintrinsicnoavx2':
1086         case 'jitx86hwintrinsicnosimd':
1087         case 'corefx_baseline':
1088         case 'corefx_minopts':
1089         case 'corefx_jitstress1':
1090         case 'corefx_jitstress2':
1091         case 'corefx_jitstressregs1':
1092         case 'corefx_jitstressregs2':
1093         case 'corefx_jitstressregs3':
1094         case 'corefx_jitstressregs4':
1095         case 'corefx_jitstressregs8':
1096         case 'corefx_jitstressregs0x10':
1097         case 'corefx_jitstressregs0x80':
1098         case 'corefx_jitstressregs0x1000':
1099         case 'zapdisable':
1100             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1101                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1102                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1103                     if (os == 'Windows_NT') {
1104                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1105                         addPeriodicTriggerHelper(job, '@weekly')
1106                     }
1107                 }
1108                 else {
1109                     addPeriodicTriggerHelper(job, '@daily')
1110                 }
1111             }
1112             break
1113         case 'heapverify1':
1114         case 'gcstress0x3':
1115             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1116                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1117                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1118                     if (os == 'Windows_NT') {
1119                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1120                         addPeriodicTriggerHelper(job, '@weekly')
1121                     }
1122                     // TODO: Add once external email sending is available again
1123                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1124                 }
1125                 else {
1126                     addPeriodicTriggerHelper(job, '@weekly')
1127                 }
1128             }
1129             break
1130         case 'gcstress0xc':
1131         case 'gcstress0xc_zapdisable':
1132         case 'gcstress0xc_zapdisable_jitstress2':
1133         case 'gcstress0xc_zapdisable_heapverify1':
1134         case 'gcstress0xc_jitstress1':
1135         case 'gcstress0xc_jitstress2':
1136         case 'gcstress0xc_minopts_heapverify1':
1137             // GCStress=C is currently not supported on OS X
1138             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1139                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1140                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1141                     if (os == 'Windows_NT') {
1142                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1143                         addPeriodicTriggerHelper(job, '@weekly')
1144                     }
1145                     // TODO: Add once external email sending is available again
1146                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1147                 }
1148                 else {
1149                     addPeriodicTriggerHelper(job, '@weekly')
1150                 }
1151             }
1152             break
1153
1154         case 'illink':
1155             // Testing on other operating systems TBD
1156             assert (os == 'Windows_NT' || os == 'Ubuntu')
1157             if (architecture == 'x64' || architecture == 'x86') {
1158                 if (configuration == 'Checked') {
1159                     addPeriodicTriggerHelper(job, '@daily')
1160                 }
1161             }
1162             break
1163
1164         case 'tieredcompilation':
1165         case 'corefx_tieredcompilation':
1166             // No periodic jobs just yet, still testing
1167             break
1168
1169         default:
1170             println("Unknown scenario: ${scenario}");
1171             assert false
1172             break
1173     }
1174     return
1175 }
1176
1177 // **************************
1178 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1179 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1180 // builds will do a couple extra steps.
1181 // **************************
1182
1183 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1184 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1185 // it's just used for internal builds
1186 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1187 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1188     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1189
1190     if (isWindowsBuildOnlyJob) {
1191         return
1192     }
1193
1194     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1195     // Non pull request builds.
1196     if (!isPR) {
1197         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1198         return
1199     }
1200
1201      def arm64Users = [
1202         'AndyAyersMS',
1203         'briansull',
1204         'BruceForstall',
1205         'CarolEidt',
1206         'cmckinsey',
1207         'erozenfeld',
1208         'janvorli',
1209         'jashook',
1210         'JosephTremoulet',
1211         'pgodeq',
1212         'pgavlin',
1213         'rartemev',
1214         'russellhadley',
1215         'RussKeldorph',
1216         'sandreenko',
1217         'sdmaclea',
1218         'swaroop-sridhar',
1219         'jkotas',
1220         'markwilkie',
1221         'weshaggard'
1222     ]
1223
1224     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1225     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1226     def osGroup = getOSGroup(os)
1227     switch (architecture) {
1228         case 'x64': // editor brace matching: {
1229             if (scenario == 'formatting') {
1230                 assert configuration == 'Checked'
1231                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1232                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1233                 }
1234
1235                 break
1236             }
1237
1238             switch (os) {
1239                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1240                 case 'Debian8.4':
1241                 case 'RHEL7.2':
1242                     if (scenario == 'innerloop') {
1243                         assert !isFlowJob
1244                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1245                     }
1246                     else if (scenario == 'normal') {
1247                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1248                     }
1249                     break
1250
1251                 case 'Ubuntu16.04':
1252                     assert !isFlowJob
1253                     assert scenario != 'innerloop'
1254                     // Distinguish with the other architectures (arm and x86)
1255                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1256                     break
1257
1258                 case 'Fedora24':
1259                 case 'Ubuntu16.10':
1260                     assert !isFlowJob
1261                     assert scenario != 'innerloop'
1262                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1263                     break
1264
1265                 case 'Ubuntu':
1266                     if (scenario == 'illink') {
1267                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1268                         break
1269                     }
1270                     // fall through
1271
1272                 case 'OSX10.12':
1273                     // Triggers on the non-flow jobs aren't necessary here
1274                     // Corefx testing uses non-flow jobs.
1275                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1276                         break
1277                     }
1278                     switch (scenario) {
1279                         case 'innerloop':
1280                             // PR Triggered jobs. These jobs will run pri0 tests.
1281                             if (configuration == 'Checked') {
1282                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test.*")
1283                             }
1284                             break
1285
1286                         case 'normal':
1287                             // OSX uses checked for default PR tests
1288                             if (configuration == 'Checked') {
1289                                 // Default trigger
1290                                 assert !job.name.contains("centos")
1291                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1292                             }
1293                             break
1294
1295                         case 'jitdiff':
1296                             if (configuration == 'Checked') {
1297                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1298                             }
1299                             break
1300
1301                         case 'ilrt':
1302                             if (configuration == 'Release') {
1303                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1304                             }
1305                             break
1306
1307                         case 'longgc':
1308                             if (configuration == 'Release') {
1309                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1310                             }
1311                             break
1312
1313                         case 'gcsimulator':
1314                             if (configuration == 'Release') {
1315                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1316                             }
1317                             break
1318
1319                         case 'standalone_gc':
1320                             if (configuration == 'Release' || configuration == 'Checked') {
1321                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1322                             }
1323                             break
1324
1325                         case 'gc_reliability_framework':
1326                             if (configuration == 'Release' || configuration == 'Checked') {
1327                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1328                             }
1329                             break
1330
1331                         default:
1332                             if (isJitStressScenario(scenario)) {
1333                                 def displayStr = getStressModeDisplayName(scenario)
1334                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1335                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1336                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1337                             }
1338                             else if (isR2RScenario(scenario)) {
1339                                 if (configuration == 'Release' || configuration == 'Checked') {
1340                                     def displayStr = getR2RDisplayName(scenario)
1341                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1342                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1343                                 }
1344                             }
1345                             else {
1346                                 println("Unknown scenario: ${scenario}");
1347                                 assert false
1348                             }
1349                             break
1350
1351                     }
1352                     break
1353
1354                 case 'CentOS7.1':
1355                     switch (scenario) {
1356                         case 'innerloop':
1357                             // CentOS uses checked for default PR tests while debug is build only
1358                             if (configuration == 'Debug') {
1359                                 // Default trigger
1360                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1361                             }
1362
1363                             // Make sure this is a flow job to get build and test.
1364                             if (configuration == 'Checked' && isFlowJob) {
1365                                 assert job.name.contains("flow")
1366                                 // Default trigger
1367                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1368                             }
1369                             break
1370
1371                         case 'normal':
1372                             // Make sure this is a flow job to get build and test.
1373                             if (configuration == 'Checked' && isFlowJob) {
1374                                 assert job.name.contains("flow")
1375                                 // Default trigger
1376                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1377                             }
1378                             break
1379
1380                         default:
1381                             if (isR2RScenario(scenario)) {
1382                                 if (configuration == 'Release' || configuration == 'Checked') {
1383                                     def displayStr = getR2RDisplayName(scenario)
1384                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1385                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1386                                 }
1387                             }
1388                             break
1389
1390                     }
1391                     break
1392
1393                 case 'Windows_NT':
1394                     switch (scenario) {
1395                         case 'innerloop':
1396                             // Default trigger
1397                             if (configuration == 'Checked' || configuration == 'Release') {
1398                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1399                             }
1400                             break
1401
1402                         case 'normal':
1403                             if (configuration == 'Checked') {
1404                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1405                             }
1406                             break
1407
1408                         case 'jitdiff':
1409                             if (configuration == 'Checked') {
1410                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1411                             }
1412                             break
1413
1414                         case 'ilrt':
1415                             if (configuration == 'Release') {
1416                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1417                             }
1418                             break
1419
1420                         case 'longgc':
1421                             if (configuration == 'Release') {
1422                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1423                             }
1424                             break
1425
1426                         case 'gcsimulator':
1427                             if (configuration == 'Release') {
1428                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1429                             }
1430                             break
1431
1432                         case 'standalone_gc':
1433                             if (configuration == 'Release' || configuration == 'Checked') {
1434                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1435                             }
1436                             break
1437
1438                         case 'gc_reliability_framework':
1439                             if (configuration == 'Release' || configuration == 'Checked') {
1440                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1441                             }
1442                             break
1443
1444                         case 'illink':
1445                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1446                             break
1447
1448                         default:
1449                             if (isJitStressScenario(scenario)) {
1450                                 def displayStr = getStressModeDisplayName(scenario)
1451                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1452                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1453                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1454                             }
1455                             else if (isR2RScenario(scenario)) {
1456                                 if (configuration == 'Release' || configuration == 'Checked') {
1457                                     def displayStr = getR2RDisplayName(scenario)
1458                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1459                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1460                                 }
1461                             }
1462                             else {
1463                                 println("Unknown scenario: ${scenario}");
1464                                 assert false
1465                             }
1466                             break
1467
1468                     }
1469                     break
1470
1471                 default:
1472                     println("Unknown os: ${os}");
1473                     assert false
1474                     break
1475
1476             }
1477
1478             break
1479
1480         // editor brace matching: }
1481         case 'armlb':
1482         case 'arm': // editor brace matching: {
1483             switch (os) {
1484                 case 'Ubuntu':
1485                 case 'Ubuntu16.04':
1486                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1487                         break
1488                     }
1489
1490                     assert scenario != 'innerloop'
1491                     job.with {
1492                         publishers {
1493                             azureVMAgentPostBuildAction {
1494                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1495                             }
1496                         }
1497                     }
1498                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1499                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1500                     break
1501
1502                 case 'Tizen':
1503                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1504                         break
1505                     }
1506
1507                     architecture = 'armel'
1508                     job.with {
1509                         publishers {
1510                             azureVMAgentPostBuildAction {
1511                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1512                             }
1513                         }
1514                     }
1515
1516                     if (scenario == 'innerloop') {
1517                         if (configuration == 'Checked') {
1518                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1519                         }
1520                     }
1521                     else {
1522                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1523                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1524                     }
1525                     break
1526
1527                 case 'Windows_NT':
1528                     // Triggers on the non-flow jobs aren't necessary here
1529                     if (!isFlowJob) {
1530                         break
1531                     }
1532
1533                     // Set up a private trigger
1534                     def contextString = "${os} ${architecture} Cross ${configuration}"
1535                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1536                     if (scenario == 'innerloop') {
1537                         contextString += " Innerloop"
1538                         triggerString += "\\W+Innerloop"
1539                     }
1540                     else {
1541                         contextString += " ${scenario}"
1542                         triggerString += "\\W+${scenario}"
1543                     }
1544
1545                     if (configuration == 'Debug') {
1546                         contextString += " Build"
1547                         triggerString += "\\W+Build"
1548                     } else {
1549                         contextString += " Build and Test"
1550                         triggerString += "\\W+Build and Test"
1551                     }
1552
1553                     triggerString += ".*"
1554
1555                     switch (scenario) {
1556                         case 'innerloop':
1557                             // Only Checked is an innerloop trigger.
1558                             if (configuration == 'Checked')
1559                             {
1560                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1561                             }
1562                             break
1563                         case 'normal':
1564                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1565                             break
1566                         default:
1567                             // Stress jobs will use this code path.
1568                             if (isArmWindowsScenario(scenario)) {
1569                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1570                             }
1571                             break
1572                     }
1573                     break
1574                 default:
1575                     println("NYI os: ${os}");
1576                     assert false
1577                     break
1578             }
1579             break
1580         // editor brace matching: }
1581         case 'arm64': // editor brace matching: {
1582             // Set up a private trigger
1583             def contextString = "${os} ${architecture} Cross ${configuration}"
1584             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1585
1586             if (scenario == 'innerloop') {
1587                 contextString += " Innerloop"
1588                 triggerString += "\\W+Innerloop"
1589             }
1590             else {
1591                 contextString += " ${scenario}"
1592                 triggerString += "\\W+${scenario}"
1593             }
1594
1595             if (configuration == 'Debug') {
1596                 contextString += " Build"
1597                 triggerString += "\\W+Build"
1598             } else {
1599                 contextString += " Build and Test"
1600                 triggerString += "\\W+Build and Test"
1601             }
1602
1603             triggerString += ".*"
1604
1605             switch (os) {
1606                 case 'Ubuntu':
1607                 case 'Ubuntu16.04':
1608                     switch (scenario) {
1609                         case 'innerloop':
1610                             if (configuration == 'Debug' && !isFlowJob) {
1611                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1612                             }
1613
1614                             break
1615                         case 'normal':
1616                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1617                             break
1618                         default:
1619                             if (isR2RScenario(scenario)) {
1620                                 if (configuration == 'Checked' || configuration == 'Release') {
1621                                     def displayStr = getR2RDisplayName(scenario)
1622                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1623                                 }
1624                             }
1625                             break
1626                     }
1627                     break
1628
1629                 case 'Windows_NT':
1630                     // Triggers on the non-flow jobs aren't necessary here
1631                     if (!isFlowJob) {
1632                         break
1633                     }
1634
1635                     assert isArmWindowsScenario(scenario)
1636                     switch (scenario) {
1637                         case 'innerloop':
1638                             if (configuration == 'Checked') {
1639                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1640                             }
1641
1642                             break
1643                         case 'normal':
1644                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1645                             break
1646                         default:
1647                             // Stress jobs will use this code path.
1648                             if (isArmWindowsScenario(scenario)) {
1649                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1650                             }
1651                             break
1652                     }
1653                     break
1654                 default:
1655                     println("NYI os: ${os}");
1656                     assert false
1657                     break
1658             }
1659             break
1660
1661         // editor brace matching: }
1662         case 'x86': // editor brace matching: {
1663             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1664             if (os == 'Ubuntu') {
1665                 // Triggers on the non-flow jobs aren't necessary here
1666                 if (!isFlowJob) {
1667                     break
1668                 }
1669
1670                 // on-demand only for ubuntu x86
1671                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1672                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1673                 break
1674
1675             }
1676             switch (scenario) {
1677                 case 'innerloop':
1678                     if (configuration == 'Checked' || configuration == 'Release') {
1679                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1680                     }
1681                     break
1682
1683                 case 'normal':
1684                     if (configuration == 'Checked') {
1685                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1686                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1687                     }
1688                     break
1689
1690                 case 'ilrt':
1691                     if (configuration == 'Release') {
1692                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1693                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1694                     }
1695                     break
1696
1697                 case 'longgc':
1698                     if (configuration == 'Release') {
1699                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1700                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1701                     }
1702                     break
1703
1704                 case 'gcsimulator':
1705                     if (configuration == 'Release') {
1706                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1707                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1708                     }
1709                     break
1710
1711                 case 'standalone_gc':
1712                     if (configuration == 'Release' || configuration == 'Checked') {
1713                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1714                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1715                     }
1716                     break
1717
1718                 case 'illink':
1719                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1720                     break
1721
1722                 default:
1723                     if (isJitStressScenario(scenario)) {
1724                         def displayStr = getStressModeDisplayName(scenario)
1725                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1726                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1727                     }
1728                     else if (isR2RScenario(scenario)) {
1729                         if (configuration == 'Release' || configuration == 'Checked') {
1730                             def displayStr = getR2RDisplayName(scenario)
1731                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1732                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1733                         }
1734                     }
1735                     else {
1736                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1737                         assert false
1738                     }
1739                     break
1740
1741             }
1742             break
1743
1744          // editor brace matching: }
1745         case 'x64_arm64_altjit':
1746         case 'x86_arm_altjit': // editor brace matching: {
1747             assert (os == 'Windows_NT')
1748             switch (scenario) {
1749                 case 'normal':
1750                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1751                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1752                     break
1753                 default:
1754                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1755                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1756                     break
1757             }
1758             break
1759
1760         // editor brace matching: }
1761         default:
1762             println("Unknown architecture: ${architecture}");
1763             assert false
1764             break
1765     }
1766 }
1767
1768 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1769     def buildCommands = [];
1770     def osGroup = getOSGroup(os)
1771     def lowerConfiguration = configuration.toLowerCase()
1772
1773     def priority = '1'
1774     if (scenario == 'innerloop') {
1775         priority = '0'
1776     }
1777
1778     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1779
1780     def enableCorefxTesting = isCoreFxScenario(scenario)
1781
1782     // Calculate the build steps, archival, and xunit results
1783     switch (os) {
1784         case 'Windows_NT': // editor brace matching: {
1785             switch (architecture) {
1786                 case 'x64':
1787                 case 'x86':
1788                 case 'x86_arm_altjit':
1789                 case 'x64_arm64_altjit':
1790                     def arch = architecture
1791                     def buildOpts = ''
1792                     if (architecture == 'x86_arm_altjit') {
1793                         arch = 'x86'
1794                     }
1795                     else if (architecture == 'x64_arm64_altjit') {
1796                         arch = 'x64'
1797                     }
1798
1799                     if (scenario == 'formatting') {
1800                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1801                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1802                         break
1803                     }
1804
1805                     if (scenario == 'illink') {
1806                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1807                     }
1808
1809                     // If it is a release build for windows, ensure PGO is used, else fail the build
1810                     if ((lowerConfiguration == 'release') &&
1811                         (scenario in Constants.basicScenarios) &&
1812                         (architecture != 'x86_arm_altjit') &&
1813                         (architecture != 'x64_arm64_altjit')) {
1814
1815                         buildOpts += ' -enforcepgo'
1816                     }
1817
1818                     if (enableCorefxTesting) {
1819                         buildOpts += ' skiptests';
1820                     } else {
1821                         buildOpts += " -priority=${priority}"
1822                     }
1823
1824                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1825                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1826                     // 35 characters long.
1827
1828                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1829
1830                     if (!isBuildOnly) {
1831                         def runtestArguments = ''
1832                         def testOpts = 'collectdumps'
1833
1834                         if (isR2RScenario(scenario)) {
1835
1836                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1837                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1838                             // to cause the tests to be crossgen'ed.
1839
1840                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1841                                 testOpts += ' crossgenaltjit protononjit.dll'
1842                             } else {
1843                                 testOpts += ' crossgen'
1844                             }
1845
1846                             testOpts += ' runcrossgentests'
1847                         }
1848                         else if (scenario == 'jitdiff') {
1849                             testOpts += ' jitdisasm crossgen'
1850                         }
1851                         else if (scenario == 'ilrt') {
1852                             testOpts += ' ilasmroundtrip'
1853                         }
1854                         else if (isLongGc(scenario)) {
1855                             testOpts += " ${scenario} sequential"
1856                         }
1857                         else if (scenario == 'standalone_gc') {
1858                             testOpts += ' gcname clrgc.dll'
1859                         }
1860                         else if (scenario == 'illink') {
1861                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1862                         }
1863
1864                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1865                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1866                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1867                         // which is read by the xunit harness.)
1868                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1869                         {
1870                             def timeout = 1800000
1871                             testOpts += " timeout ${timeout}"
1872                         }
1873
1874                         // If we are running a stress mode, we should write out the set of key
1875                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1876
1877                         def envScriptPath = ''
1878                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1879                             def buildCommandsStr = ''
1880                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1881                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1882
1883                             if (isJitStressScenario(scenario)) {
1884                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1885                             }
1886                             else if (isR2RStressScenario(scenario)) {
1887                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1888                             }
1889
1890                             if (architecture == 'x86_arm_altjit') {
1891                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1892                             }
1893                             else if (architecture == 'x64_arm64_altjit') {
1894                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1895                             }
1896
1897                             envScriptFinalize(os, envScriptPath)
1898
1899                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1900                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1901                             // into a single script.
1902                             buildCommands += buildCommandsStr
1903                         }
1904                         else if (architecture == 'x86_arm_altjit') {
1905                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1906                         }
1907                         else if (architecture == 'x64_arm64_altjit') {
1908                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1909                         }
1910                         if (envScriptPath != '') {
1911                             testOpts += " TestEnv ${envScriptPath}"
1912                         }
1913
1914                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1915
1916                         if (enableCorefxTesting) {
1917                             def workspaceRelativeFxRoot = "_/fx"
1918                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1919
1920                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1921
1922                             // Archive and process (only) the test results
1923                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1924                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1925
1926                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1927                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1928                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1929                         }
1930                         else if (isGcReliabilityFramework(scenario)) {
1931                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1932                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1933                         }
1934                         else {
1935                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1936                         }
1937                     }
1938
1939                     if (!enableCorefxTesting) {
1940                         // Run the rest of the build
1941                         // Build the mscorlib for the other OS's
1942                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1943                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1944
1945                         if (arch == "x64") {
1946                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1947                         }
1948
1949                         // Zip up the tests directory so that we don't use so much space/time copying
1950                         // 10s of thousands of files around.
1951                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1952
1953                         if (!isJitStressScenario(scenario)) {
1954                             // For windows, pull full test results and test drops for x86/x64.
1955                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1956                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1957                         }
1958
1959                         if (scenario == 'jitdiff') {
1960                             // retrive jit-dasm output for base commit, and run jit-diff
1961                             if (!isBuildOnly) {
1962                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1963                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1964                             }
1965                         }
1966
1967                         if (!isBuildOnly) {
1968                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1969                         }
1970                     }
1971                     break
1972                 case 'armlb':
1973                 case 'arm':
1974                     assert isArmWindowsScenario(scenario)
1975
1976                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1977                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1978
1979                     def buildArchitecture = 'arm'
1980
1981                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1982                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1983                     def armCrossgenOpt = ''
1984                     if (architecture == 'armlb') {
1985                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1986                     }
1987
1988                     // This is now a build only job. Do not run tests. Use the flow job.
1989                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1990
1991                     // Zip up the tests directory so that we don't use so much space/time copying
1992                     // 10s of thousands of files around.
1993                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1994
1995                     // Add archival.
1996                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1997                     break
1998                 case 'arm64':
1999                     assert isArmWindowsScenario(scenario)
2000
2001                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2002                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2003
2004                     // This is now a build only job. Do not run tests. Use the flow job.
2005                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2006
2007                     // Zip up the tests directory so that we don't use so much space/time copying
2008                     // 10s of thousands of files around.
2009                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2010
2011                     // Add archival.
2012                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2013                     break
2014                 default:
2015                     println("Unknown architecture: ${architecture}");
2016                     assert false
2017                     break
2018             }
2019             break
2020         // editor brace matching: }
2021         case 'Ubuntu':
2022         case 'Ubuntu16.04':
2023         case 'Ubuntu16.10':
2024         case 'Debian8.4':
2025         case 'OSX10.12':
2026         case 'CentOS7.1':
2027         case 'RHEL7.2':
2028         case 'Tizen':
2029         case 'Fedora24': // editor brace matching: {
2030             switch (architecture) {
2031                 case 'x64':
2032                 case 'x86':
2033                     if (architecture == 'x86' && os == 'Ubuntu') {
2034                         // build and PAL test
2035                         def dockerImage = getDockerImageName(architecture, os, true)
2036                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2037                         dockerImage = getDockerImageName(architecture, os, false)
2038                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2039                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2040                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2041                         break
2042                     }
2043
2044                     if (scenario == 'formatting') {
2045                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2046                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2047                         break
2048                     }
2049
2050                     if (scenario == 'illink') {
2051                         assert(os == 'Ubuntu')
2052                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2053                     }
2054
2055                     if (!enableCorefxTesting) {
2056                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2057                         // only on supported OS platforms.
2058                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2059                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2060
2061                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2062                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2063
2064                         // Basic archiving of the build
2065                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2066                         // And pal tests
2067                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2068                     }
2069                     else {
2070                         // Corefx stress testing
2071                         assert os == 'Ubuntu'
2072                         assert architecture == 'x64'
2073                         assert lowerConfiguration == 'checked'
2074                         assert isJitStressScenario(scenario)
2075
2076                         // Build coreclr
2077                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2078
2079                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2080
2081                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2082                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2083                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2084                         buildCommands += envScriptCmds
2085
2086                         // Build and text corefx
2087                         def workspaceRelativeFxRoot = "_/fx"
2088                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2089
2090                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2091
2092                         // Archive and process (only) the test results
2093                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2094                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2095                     }
2096                     break
2097                 case 'arm64':
2098                     if (!enableCorefxTesting) {
2099                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2100
2101                         // HACK -- Arm64 does not have corefx jobs yet.
2102                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2103                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2104                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2105                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2106
2107                         // Basic archiving of the build
2108                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2109                     }
2110                     break
2111                 case 'arm':
2112                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2113                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2114
2115                     // default values for Ubuntu
2116                     def arm_abi="arm"
2117                     def linuxCodeName="trusty"
2118                     if (os == 'Ubuntu16.04') {
2119                         linuxCodeName="xenial"
2120                     }
2121                     else if (os == 'Tizen') {
2122                         arm_abi="armel"
2123                         linuxCodeName="tizen"
2124                     }
2125
2126                     // Unzip the Windows test binaries first. Exit with 0
2127                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2128
2129                     // Unpack the corefx binaries
2130                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2131                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2132                     if (os != 'Tizen') {
2133                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2134                     }
2135                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2136                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2137                     if (os == 'Ubuntu') {
2138                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2139                     }
2140
2141                     // Call the ARM CI script to cross build and test using docker
2142                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2143                     --mode=docker \\
2144                     --${arm_abi} \\
2145                     --linuxCodeName=${linuxCodeName} \\
2146                     --buildConfig=${lowerConfiguration} \\
2147                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2148                     --coreFxBinDir=./bin/CoreFxBinDir \\
2149                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2150
2151                     // Basic archiving of the build, no pal tests
2152                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2153                     break
2154                 default:
2155                     println("Unknown architecture: ${architecture}");
2156                     assert false
2157                     break
2158             }
2159             break
2160         // editor brace matching: }
2161         default:
2162             println("Unknown os: ${os}");
2163             assert false
2164             break
2165     } // os
2166
2167     return buildCommands
2168 }
2169
2170 Constants.allScenarios.each { scenario ->
2171     [true, false].each { isPR ->
2172         Constants.architectureList.each { architecture ->
2173             Constants.configurationList.each { configuration ->
2174                 Constants.osList.each { os ->
2175                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2176                     // and reset the os to Windows_NT
2177                     def isBuildOnly = false
2178                     if (os == 'Windows_NT_BuildOnly') {
2179                         isBuildOnly = true
2180                         os = 'Windows_NT'
2181                     }
2182
2183                     // Tizen is only supported for arm architecture
2184                     if (os == 'Tizen' && architecture != 'arm') {
2185                         return
2186                     }
2187
2188                     // Skip totally unimplemented (in CI) configurations.
2189                     switch (architecture) {
2190                         case 'arm64':
2191                             if (os == 'Ubuntu16.04') {
2192                                 os = 'Ubuntu'
2193                             }
2194
2195                             // Windows and Ubuntu only
2196                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2197                                 return
2198                             }
2199                             break
2200                         case 'arm':
2201                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2202                                 return
2203                             }
2204                             break
2205                         case 'armlb':
2206                             if (os != 'Windows_NT') {
2207                                 return
2208                             }
2209                             break
2210                         case 'x86':
2211                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2212                                 return
2213                             }
2214                             break
2215                         case 'x86_arm_altjit':
2216                         case 'x64_arm64_altjit':
2217                             if (os != 'Windows_NT') {
2218                                 return
2219                             }
2220                             break
2221                         case 'x64':
2222                             // Everything implemented
2223                             break
2224                         default:
2225                             println("Unknown architecture: ${architecture}")
2226                             assert false
2227                             break
2228                     }
2229
2230                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2231                     // with checked builds)
2232                     if (isJitStressScenario(scenario)) {
2233                         if (configuration != 'Checked') {
2234                             return
2235                         }
2236
2237                         // Since these are just execution time differences,
2238                         // skip platforms that don't execute the tests here (Windows_NT only)
2239                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2240                         if (!isEnabledOS || isBuildOnly) {
2241                             return
2242                         }
2243
2244                         switch (architecture) {
2245                             case 'x64':
2246                             case 'x86':
2247                             case 'x86_arm_altjit':
2248                             case 'x64_arm64_altjit':
2249                                 // x86 ubuntu: default only
2250                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2251                                     return
2252                                 }
2253                                 // Windows: Everything implemented
2254                                 break
2255
2256                             default:
2257                                 // arm, arm64, armlb: stress is handled through flow jobs.
2258                                 return
2259                         }
2260                     }
2261                     else if (isR2RScenario(scenario)) {
2262                         if (os != 'Windows_NT') {
2263                             return
2264                         }
2265                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2266                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2267                             return
2268                         }
2269                     }
2270                     else {
2271                         // Skip scenarios
2272                         switch (scenario) {
2273                             case 'ilrt':
2274                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2275                                 // the default scenario build
2276                                 if (os != 'Windows_NT') {
2277                                     return
2278                                 }
2279                                 // Only x64 for now
2280                                 if (architecture != 'x64') {
2281                                     return
2282                                 }
2283                                 // Release only
2284                                 if (configuration != 'Release') {
2285                                     return
2286                                 }
2287                                 break
2288                             case 'jitdiff':
2289                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2290                                     return
2291                                 }
2292                                 if (architecture != 'x64') {
2293                                     return
2294                                 }
2295                                 if (configuration != 'Checked') {
2296                                     return
2297                                 }
2298                                 break
2299                             case 'longgc':
2300                             case 'gcsimulator':
2301                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2302                                     return
2303                                 }
2304                                 if (architecture != 'x64') {
2305                                     return
2306                                 }
2307                                 if (configuration != 'Release') {
2308                                     return
2309                                 }
2310                                 break
2311                             case 'gc_reliability_framework':
2312                             case 'standalone_gc':
2313                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2314                                     return
2315                                 }
2316
2317                                 if (architecture != 'x64') {
2318                                     return
2319                                 }
2320
2321                                 if (configuration != 'Release' && configuration != 'Checked') {
2322                                     return
2323                                 }
2324                                 break
2325                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2326                             case 'formatting':
2327                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2328                                     return
2329                                 }
2330                                 if (architecture != 'x64') {
2331                                     return
2332                                 }
2333                                 if (configuration != 'Checked') {
2334                                     return
2335                                 }
2336                                 if (isBuildOnly) {
2337                                     return
2338                                 }
2339                                 break
2340                             case 'illink':
2341                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2342                                     return
2343                                 }
2344                                 if (architecture != 'x64' && architecture != 'x86') {
2345                                     return
2346                                 }
2347                                 if (isBuildOnly) {
2348                                     return
2349                                 }
2350                                 break
2351                             case 'normal':
2352                                 // Nothing skipped
2353                                 break
2354                             case 'innerloop':
2355                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2356                                     return
2357                                 }
2358                                 break
2359                             default:
2360                                 println("Unknown scenario: ${scenario}")
2361                                 assert false
2362                                 break
2363                         }
2364                     }
2365
2366                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2367                     // runtime behavior, not compile-time behavior, are not interesting.
2368                     switch (architecture) {
2369                         case 'x86_arm_altjit':
2370                         case 'x64_arm64_altjit':
2371                             if (isGCStressRelatedTesting(scenario)) {
2372                                 return
2373                             }
2374                             break
2375                         default:
2376                             break
2377                     }
2378
2379                     // Calculate names
2380                     def lowerConfiguration = configuration.toLowerCase()
2381                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2382                     def folderName = getJobFolder(scenario)
2383
2384                     // Create the new job
2385                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2386                     addToViews(newJob, isPR, architecture, os)
2387
2388                     def machineAffinityOptions = null
2389
2390                     if (os != 'Windows_NT') {
2391                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2392                     }
2393                     else {
2394                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2395                     }
2396
2397                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2398
2399                     // Add all the standard options
2400                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2401                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2402
2403                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2404                     def osGroup = getOSGroup(os)
2405
2406                     newJob.with {
2407                         steps {
2408                             if (os == 'Windows_NT') {
2409                                 buildCommands.each { buildCommand ->
2410                                     batchFile(buildCommand)
2411                                 }
2412                             }
2413                             else {
2414                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2415                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2416                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2417                                     // Define the Windows Tests and Corefx build job names
2418                                     def WindowsTestsName = projectFolder + '/' +
2419                                                            Utilities.getFullJobName(project,
2420                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2421                                                                                     false)
2422                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2423                                                        Utilities.getFolderName(branch)
2424
2425                                     // Copy the Windows test binaries and the Corefx build binaries
2426                                     copyArtifacts(WindowsTestsName) {
2427                                         includePatterns('bin/tests/tests.zip')
2428                                         buildSelector {
2429                                             latestSuccessful(true)
2430                                         }
2431                                     }
2432
2433                                     def arm_abi = 'arm'
2434                                     def corefx_os = 'linux'
2435                                     if (os == 'Tizen') {
2436                                         arm_abi = 'armel'
2437                                         corefx_os = 'tizen'
2438                                     }
2439
2440                                     // Let's use release CoreFX to test checked CoreCLR,
2441                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2442                                     def corefx_lowerConfiguration = lowerConfiguration
2443                                     if ( lowerConfiguration == 'checked' ) {
2444                                         corefx_lowerConfiguration='release'
2445                                     }
2446
2447                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2448                                         includePatterns('bin/build.tar.gz')
2449                                         buildSelector {
2450                                             latestSuccessful(true)
2451                                         }
2452                                     }
2453                                 }
2454
2455                                 buildCommands.each { buildCommand ->
2456                                     shell(buildCommand)
2457                                 }
2458                             }
2459                         }
2460                     } // newJob.with
2461
2462                 } // os
2463             } // configuration
2464         } // architecture
2465     } // isPR
2466 } // scenario
2467
2468
2469 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2470 Constants.allScenarios.each { scenario ->
2471     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2472
2473     [true, false].each { isPR ->
2474         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2475             Constants.crossList.each { os ->
2476                 if (architecture == 'arm64') {
2477                     if (os != "Ubuntu" && os != "Windows_NT") {
2478                         return
2479                     }
2480                 } else if (architecture == 'arm' || architecture == 'armlb') {
2481                     if (os != 'Windows_NT') {
2482                         return
2483                     }
2484                 } else if (architecture == 'x86') {
2485                     if (os != "Ubuntu") {
2486                         return
2487                     }
2488                 }
2489
2490                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2491
2492                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2493                     return
2494                 }
2495
2496                 Constants.configurationList.each { configuration ->
2497
2498                     // First, filter based on OS.
2499
2500                     if (os == 'Windows_NT') {
2501                         if (!isArmWindowsScenario(scenario)) {
2502                             return
2503                         }
2504                     }
2505                     else {
2506                         // Non-Windows
2507                         if (architecture == 'arm64') {
2508                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2509                                 return
2510                             }
2511                         }
2512                         else if (architecture == 'x86') {
2513                             // Linux/x86 only want innerloop and default test
2514                             if (!isNormalOrInnerloop) {
2515                                 return
2516                             }
2517                         }
2518                     }
2519
2520                     // For CentOS, we only want Checked/Release builds.
2521                     if (os == 'CentOS7.1') {
2522                         if (configuration != 'Checked' && configuration != 'Release') {
2523                             return
2524                         }
2525                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2526                             return
2527                         }
2528                     }
2529
2530                     // For RedHat and Debian, we only do Release builds.
2531                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2532                         if (configuration != 'Release') {
2533                             return
2534                         }
2535                         if (!isNormalOrInnerloop) {
2536                             return
2537                         }
2538                     }
2539
2540                     // Next, filter based on scenario.
2541
2542                     if (isJitStressScenario(scenario)) {
2543                         if (configuration != 'Checked') {
2544                             return
2545                         }
2546                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2547                         if (isCoreFxScenario(scenario)) {
2548                             return
2549                         }
2550                     }
2551                     else if (isR2RBaselineScenario(scenario)) {
2552                         if (configuration != 'Checked' && configuration != 'Release') {
2553                             return
2554                         }
2555                     }
2556                     else if (isR2RStressScenario(scenario)) {
2557                         if (configuration != 'Checked') {
2558                             return
2559                         }
2560                     }
2561                     else {
2562                         // Skip scenarios
2563                         switch (scenario) {
2564                             case 'ilrt':
2565                             case 'longgc':
2566                             case 'gcsimulator':
2567                                 // Long GC tests take a long time on non-Release builds
2568                                 // ilrt is also Release only
2569                                 if (configuration != 'Release') {
2570                                     return
2571                                 }
2572                                 break
2573
2574                             case 'jitdiff':
2575                                 if (configuration != 'Checked') {
2576                                     return;
2577                                 }
2578                                 break
2579
2580                             case 'gc_reliability_framework':
2581                             case 'standalone_gc':
2582                                 if (configuration != 'Release' && configuration != 'Checked') {
2583                                     return
2584                                 }
2585                                 break
2586
2587                             case 'formatting':
2588                                 return
2589                             case 'illink':
2590                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2591                                     return
2592                                 }
2593                                 break
2594
2595                             case 'normal':
2596                                 // Nothing skipped
2597                                 break
2598
2599                             case 'innerloop':
2600                                 // Nothing skipped
2601                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2602                                     return
2603                                 }
2604                                 break
2605
2606                             default:
2607                                 println("Unknown scenario: ${scenario}")
2608                                 assert false
2609                                 break
2610                         }
2611                     }
2612
2613                     // Done filtering. Now, create the jobs.
2614
2615                     def lowerConfiguration = configuration.toLowerCase()
2616                     def osGroup = getOSGroup(os)
2617                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2618
2619                     def buildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2620
2621                     def inputCoreCLRBuildName = projectFolder + '/' +
2622                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, buildScenario, false), isPR)
2623
2624                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2625                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2626                     def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2627                     def inputWindowsTestBuildArch = architecture
2628                     if (architecture == "arm64" && os != "Windows_NT") {
2629                         // Use the x64 test build for arm64 unix
2630                         inputWindowsTestBuildArch = "x64"
2631                     }
2632
2633                     def inputWindowsTestsBuildName = ""
2634
2635                     if (isJitStressScenario(scenario)) {
2636                         inputWindowsTestsBuildName = projectFolder + '/' +
2637                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2638                     } else {
2639                         inputWindowsTestsBuildName = projectFolder + '/' +
2640                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2641
2642                     }
2643
2644
2645                     // Enable Server GC for Ubuntu PR builds
2646                     def serverGCString = ''
2647                     if (os == 'Ubuntu' && isPR) {
2648                         serverGCString = '--useServerGC'
2649                     }
2650
2651                     def testOpts = ''
2652
2653                     if (isR2RScenario(scenario)) {
2654
2655                         testOpts += ' --crossgen --runcrossgentests'
2656
2657                         if (scenario == 'r2r_jitstress1') {
2658                             testOpts += ' --jitstress=1'
2659                         }
2660                         else if (scenario == 'r2r_jitstress2') {
2661                             testOpts += ' --jitstress=2'
2662                         }
2663                         else if (scenario == 'r2r_jitstressregs1') {
2664                             testOpts += ' --jitstressregs=1'
2665                         }
2666                         else if (scenario == 'r2r_jitstressregs2') {
2667                             testOpts += ' --jitstressregs=2'
2668                         }
2669                         else if (scenario == 'r2r_jitstressregs3') {
2670                             testOpts += ' --jitstressregs=3'
2671                         }
2672                         else if (scenario == 'r2r_jitstressregs4') {
2673                             testOpts += ' --jitstressregs=4'
2674                         }
2675                         else if (scenario == 'r2r_jitstressregs8') {
2676                             testOpts += ' --jitstressregs=8'
2677                         }
2678                         else if (scenario == 'r2r_jitstressregs0x10') {
2679                             testOpts += ' --jitstressregs=0x10'
2680                         }
2681                         else if (scenario == 'r2r_jitstressregs0x80') {
2682                             testOpts += ' --jitstressregs=0x80'
2683                         }
2684                         else if (scenario == 'r2r_jitstressregs0x1000') {
2685                             testOpts += ' --jitstressregs=0x1000'
2686                         }
2687                         else if (scenario == 'r2r_jitminopts') {
2688                             testOpts += ' --jitminopts'
2689                         }
2690                         else if (scenario == 'r2r_jitforcerelocs') {
2691                             testOpts += ' --jitforcerelocs'
2692                         }
2693                         else if (scenario == 'r2r_gcstress15') {
2694                             testOpts += ' --gcstresslevel=0xF'
2695                         }
2696                     }
2697                     else if (scenario == 'jitdiff') {
2698                         testOpts += ' --jitdisasm --crossgen'
2699                     }
2700                     else if (scenario == 'illink') {
2701                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2702                     }
2703                     else if (isLongGc(scenario)) {
2704                         // Long GC tests behave very poorly when they are not
2705                         // the only test running (many of them allocate until OOM).
2706                         testOpts += ' --sequential'
2707
2708                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2709                         // for running long GC and GCSimulator tests, respectively. We don't use them
2710                         // here because using a playlist file produces much more readable output on the CI machines
2711                         // and reduces running time.
2712                         //
2713                         // The Long GC playlist contains all of the tests that are
2714                         // going to be run. The GCSimulator playlist contains all of
2715                         // the GC simulator tests.
2716                         if (scenario == 'longgc') {
2717                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2718                         }
2719                         else if (scenario == 'gcsimulator') {
2720                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2721                         }
2722                     }
2723                     else if (isGcReliabilityFramework(scenario)) {
2724                         testOpts += ' --build-overlay-only'
2725                     }
2726                     else if (scenario == 'standalone_gc') {
2727                         if (osGroup == 'OSX') {
2728                             testOpts += ' --gcname=libclrgc.dylib'
2729                         }
2730                         else if (osGroup == 'Linux') {
2731                             testOpts += ' --gcname=libclrgc.so'
2732                         }
2733                         else {
2734                             println("Unexpected OS group: ${osGroup} for os ${os}")
2735                             assert false
2736                         }
2737                     }
2738
2739                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2740
2741                     def folder = getJobFolder(scenario)
2742                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2743                         // Add parameters for the inputs
2744
2745                         if (windowsArmJob == true) {
2746                             parameters {
2747                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2748                             }
2749                         }
2750                         else {
2751                             parameters {
2752                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2753                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2754                             }
2755                         }
2756
2757                         steps {
2758                             // Set up the copies
2759
2760                             // Coreclr build containing the tests and mscorlib
2761                             // pri1 jobs still need to copy windows_nt built tests
2762                             if (windowsArmJob != true) {
2763                                 copyArtifacts(inputWindowsTestsBuildName) {
2764                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2765                                     buildSelector {
2766                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2767                                     }
2768                                 }
2769                             }
2770
2771                             // Coreclr build we are trying to test
2772                             //
2773                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2774
2775                             copyArtifacts(inputCoreCLRBuildName) {
2776                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2777                                 buildSelector {
2778                                     buildNumber('${CORECLR_BUILD}')
2779                                 }
2780                             }
2781
2782                             // Windows CoreCLR Arm(64) will restore corefx
2783                             // packages correctly.
2784                             //
2785                             // In addition, test steps are entirely different
2786                             // because we do not have a unified runner
2787                             if (windowsArmJob != true) {
2788                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2789
2790                                 // HACK -- Arm64 does not have corefx jobs yet.
2791                                 // Clone corefx and build the native packages overwriting the x64 packages.
2792                                 if (architecture == 'arm64') {
2793                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2794                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2795                                 }
2796                                 else if (architecture == 'x86') {
2797                                     shell("mkdir ./bin/CoreFxNative")
2798
2799                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2800                                         includePatterns('bin/build.tar.gz')
2801                                         targetDirectory('bin/CoreFxNative')
2802                                         buildSelector {
2803                                             latestSuccessful(true)
2804                                         }
2805                                     }
2806
2807                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2808                                 }
2809
2810                                 // Unzip the tests first.  Exit with 0
2811                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2812                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2813
2814                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2815
2816                                 // Execute the tests
2817                                 def runDocker = isNeedDocker(architecture, os, false)
2818                                 def dockerPrefix = ""
2819                                 def dockerCmd = ""
2820                                 if (runDocker) {
2821                                     def dockerImage = getDockerImageName(architecture, os, false)
2822                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2823                                     dockerCmd = dockerPrefix + "${dockerImage} "
2824                                 }
2825
2826                                 // If we are running a stress mode, we'll set those variables first
2827                                 def testEnvOpt = ""
2828                                 if (isJitStressScenario(scenario)) {
2829                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2830                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2831                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2832                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2833                                     shell("${envScriptCmds}")
2834                                     testEnvOpt = "--test-env=" + scriptFileName
2835                                 }
2836
2837                                 if (isGCStressRelatedTesting(scenario)) {
2838                                     shell('./init-tools.sh')
2839                                 }
2840
2841                                 shell("""${dockerCmd}./tests/runtest.sh \\
2842                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2843                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2844                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2845                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2846
2847                                 if (isGcReliabilityFramework(scenario)) {
2848                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2849                                     if (serverGCString != '') {
2850                                         if (runDocker) {
2851                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2852                                         }
2853                                         else {
2854                                             shell("export COMPlus_gcServer=1")
2855                                         }
2856                                     }
2857
2858                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2859                                 }
2860                             }
2861
2862                             else { // windowsArmJob == true
2863                                 // Unzip tests.
2864                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2865
2866                                 // Build the build commands
2867                                 def buildCommands = ""
2868
2869                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2870                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2871                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2872
2873                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2874                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2875
2876                                 // For all jobs
2877                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2878
2879                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2880                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2881
2882                                 // ARM legacy backend; this is an altjit.
2883                                 if (architecture == "armlb") {
2884                                     addEnvVariable("COMPlus_AltJit", "*")
2885                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2886                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2887                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2888                                 }
2889
2890                                 // If we are running a stress mode, we'll set those variables as well
2891                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2892                                     def stressValues = null
2893                                     if (isJitStressScenario(scenario)) {
2894                                         stressValues = Constants.jitStressModeScenarios[scenario]
2895                                     }
2896                                     else {
2897                                         stressValues = Constants.r2rStressScenarios[scenario]
2898                                     }
2899
2900                                     stressValues.each { key, value ->
2901                                         addEnvVariable(key, value)
2902                                     }
2903                                 }
2904
2905                                 if (isR2RScenario(scenario)) {
2906                                     // Crossgen the framework assemblies.
2907                                     buildCommands += """
2908 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2909 @goto skip_PrecompileAssembly
2910
2911 :PrecompileAssembly
2912 @REM Skip mscorlib since it is already precompiled.
2913 @if /I "%3" == "mscorlib.dll" exit /b 0
2914 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2915
2916 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2917 @if "%errorlevel%" == "-2146230517" (
2918     echo %2 is not a managed assembly.
2919 ) else if "%errorlevel%" == "-2146234344" (
2920     echo %2 is not a managed assembly.
2921 ) else if %errorlevel% neq 0 (
2922     echo Unable to precompile %2
2923 ) else (
2924     echo Precompiled %2
2925 )
2926 @exit /b 0
2927
2928 :skip_PrecompileAssembly
2929 """
2930
2931                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2932                                     // crossgen on tests before running them.
2933                                     addEnvVariable("RunCrossGen", "true")
2934                                 }
2935
2936                                 // Create the smarty command
2937                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2938                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2939                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2940
2941                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2942
2943                                 if (architecture == "armlb") {
2944                                     addExclude("LEGACYJIT_FAIL")
2945                                 }
2946
2947                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2948                                     def failTag = "JITSTRESS_FAIL"
2949                                     def excludeTag = "JITSTRESS_EXCLUDE"
2950
2951                                     if (scenario.contains('gc')) {
2952                                         failTag = "GCSTRESS_FAIL"
2953                                         excludeTag = "GCSTRESS_EXCLUDE"
2954                                     }
2955
2956                                     addArchSpecificExclude(architecture, failTag)
2957                                     addArchSpecificExclude(architecture, excludeTag)
2958                                 }
2959                                 else {
2960                                     addExclude("pri1")
2961                                 }
2962
2963                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2964                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2965                                 addExclude("LONG_RUNNING")
2966
2967                                 smartyCommand += "/lstFile Tests.lst"
2968
2969                                 def testListArch = [
2970                                     'arm64': 'arm64',
2971                                     'arm': 'arm',
2972                                     'armlb': 'arm'
2973                                 ]
2974
2975                                 def archLocation = testListArch[architecture]
2976
2977                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2978                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2979                                 addCommand("${smartyCommand}")
2980
2981                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2982                                 // However, we also need to remove all the variables that were set during this batch file, so we
2983                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2984                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2985                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2986                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2987                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2988                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2989
2990                                 // ZIP up the smarty output, no matter what the smarty result.
2991                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2992
2993                                 addCommand("echo %errorlevel%")
2994                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2995
2996                                 // Use the smarty errorlevel as the script errorlevel.
2997                                 addCommand("exit /b %__save_smarty_errorlevel%")
2998
2999                                 batchFile(buildCommands)
3000                             }
3001                         }
3002                     }
3003
3004                     addToViews(newJob, isPR, architecture, os)
3005
3006                     if (scenario == 'jitdiff') {
3007                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3008                     }
3009
3010                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3011                     if (os in ['Ubuntu']) {
3012                         SummaryBuilder summaries = new SummaryBuilder()
3013                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3014                         summaries.emit(newJob)
3015                     }
3016
3017                     def affinityOptions = null
3018
3019                     if (windowsArmJob == true) {
3020                         affinityOptions = [
3021                             "use_arm64_build_machine" : false
3022                         ]
3023                     }
3024
3025                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3026                         affinityOptions = [
3027                             "large_pages" : false
3028                         ]
3029                     }
3030
3031                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3032                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3033
3034                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3035
3036                     if (windowsArmJob != true) {
3037                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3038                     }
3039                     else {
3040                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3041
3042                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3043                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3044                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3045                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3046                     }
3047
3048                     // Create a build flow to join together the build and tests required to run this test.
3049                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3050                     // Linux CoreCLR test
3051                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3052                     def fullTestJobName = projectFolder + '/' + newJob.name
3053                     // Add a reference to the input jobs for report purposes
3054                     JobReport.Report.addReference(inputCoreCLRBuildName)
3055                     JobReport.Report.addReference(inputWindowsTestsBuildName)
3056                     JobReport.Report.addReference(fullTestJobName)
3057                     def newFlowJob = null
3058
3059                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3060                         // Do not create the flow job for RHEL jobs.
3061                         return
3062                     }
3063
3064                     // For pri0 jobs we can build tests on unix
3065                     if (windowsArmJob) {
3066                         // For Windows arm jobs there is no reason to build a parallel test job.
3067                         // The product build supports building and archiving the tests.
3068
3069                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3070                         buildFlow("""
3071 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3072
3073 // And then build the test build
3074 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3075 """)
3076                         }
3077                     }
3078                     else {
3079                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3080                         buildFlow("""
3081 // Build the input jobs in parallel
3082 parallel (
3083 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3084 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3085 )
3086
3087 // And then build the test build
3088 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3089             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3090 """)
3091                         }
3092                     }
3093
3094                     addToViews(newFlowJob, isPR, architecture, os)
3095
3096                     // For the flow jobs set the machine affinity as x64 if an armarch.
3097                     def flowArch = architecture
3098
3099                     if (flowArch in validWindowsNTCrossArches) {
3100                         flowArch = 'x64'
3101                         affinityOptions = null
3102                     }
3103
3104                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3105                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3106                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3107                 } // configuration
3108             } // os
3109         } // architecture
3110     } // isPR
3111 } // scenario
3112
3113 JobReport.Report.generateJobReport(out)
3114
3115 // Make the call to generate the help job
3116 Utilities.createHelperJob(this, project, branch,
3117     "Welcome to the ${project} Repository",  // This is prepended to the help message
3118     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3119
3120 Utilities.addCROSSCheck(this, project, branch)