netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'innerloop',
 139                'normal',
 140                'ilrt',
 141                'r2r',
 142                'longgc',
 143                'formatting',
 144                'gcsimulator',
 145                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 146                'standalone_gc',
 147                'gc_reliability_framework',
 148                'illink']
 149
 150     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 151
 152     // Valid PR trigger combinations.
 153     def static prTriggeredValidInnerLoopCombos = [
 154         'Windows_NT': [
 155             'x64': [
 156                 'Checked'
 157             ],
 158             'x86': [
 159                 'Checked',
 160                 'Release'
 161             ],
 162             'arm': [
 163                 'Checked',
 164             ],
 165             'arm64': [
 166                 'Checked'
 167             ],
 168             'armlb': [
 169                 'Checked'
 170             ]
 171         ],
 172         'Windows_NT_buildOnly': [
 173             'x64': [
 174                 'Checked',
 175                 'Release'
 176             ],
 177             'x86': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181         ],
 182         'Ubuntu': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'arm64': [
 187                 'Debug'
 188             ]
 189         ],
 190         'CentOS7.1': [
 191             'x64': [
 192                 'Debug',
 193                 'Checked'
 194             ]
 195         ],
 196         'OSX10.12': [
 197             'x64': [
 198                 'Checked'
 199             ]
 200         ],
 201         'Tizen': [
 202             'arm': [
 203                 'Checked'
 204             ]
 205         ],
 206     ]
 207
 208     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 209     // to Tests.lst file categories to exclude.
 210     //
 211     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 212     // and with the same values, with some commented out, for easier maintenance.
 213     //
 214     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 215     //
 216     def static validArmWindowsScenarios = [
 217                'innerloop':                              [],
 218                'normal':                                 [],
 219                // 'ilrt'
 220                'r2r':                                    ["R2R_FAIL"],
 221                // 'longgc'
 222                // 'formatting'
 223                // 'gcsimulator'
 224                // 'jitdiff'
 225                // 'standalone_gc'
 226                // 'gc_reliability_framework'
 227                // 'illink'
 228                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 242                'tieredcompilation':                      [],
 243                'forcerelocs':                            [],
 244                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 263                // 'jitsse2only'                          // Only relevant to xarch
 264                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 265                // 'corefx_baseline'
 266                // 'corefx_minopts'
 267                // 'corefx_tieredcompilation'
 268                // 'corefx_jitstress1'
 269                // 'corefx_jitstress2'
 270                // 'corefx_jitstressregs1'
 271                // 'corefx_jitstressregs2'
 272                // 'corefx_jitstressregs3'
 273                // 'corefx_jitstressregs4'
 274                // 'corefx_jitstressregs8'
 275                // 'corefx_jitstressregs0x10'
 276                // 'corefx_jitstressregs0x80'
 277                // 'corefx_jitstressregs0x1000'
 278                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 279                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 280                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 281                'heapverify1':                            [],
 282                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 283                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 284                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 285                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 288
 289                //
 290                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 291                //
 292
 293                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 294                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 307                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 308                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 309                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 312     ]
 313
 314     def static validLinuxArm64Scenarios = [
 315         'normal',
 316         'r2r',
 317         'innerloop',
 318         'gcstress0x3',
 319         'gcstress0xc'
 320     ]
 321
 322     def static configurationList = ['Debug', 'Checked', 'Release']
 323
 324     // This is the set of architectures
 325     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 326 }
 327
 328 // **************************************************************
 329 // Create some specific views
 330 //
 331 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 332 // views based on a single regular expression. These views will be generated by adding a
 333 // specific set of jobs to them.
 334 //
 335 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 336 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 337 // those are required, don't add them (which simplifies the view pages, as well).
 338 // **************************************************************
 339
 340 class Views {
 341     def static MergeJobView = null
 342     def static PeriodicJobView = null
 343     def static ArchitectureViews = [:]
 344     def static OSViews = [:]
 345 }
 346
 347 // MergeJobView: include all jobs that execute when a PR change is merged.
 348 Views.MergeJobView = listView('Merge') {
 349     recurse()
 350     columns {
 351         status()
 352         weather()
 353         name()
 354         lastSuccess()
 355         lastFailure()
 356         lastDuration()
 357         buildButton()
 358     }
 359 }
 360
 361 // PeriodicJobView: include all jobs that execute on a schedule
 362 Views.PeriodicJobView = listView('Periodic') {
 363     recurse()
 364     columns {
 365         status()
 366         weather()
 367         name()
 368         lastSuccess()
 369         lastFailure()
 370         lastDuration()
 371         buildButton()
 372     }
 373 }
 374
 375 // Create a view for non-PR jobs for each architecture.
 376 Constants.architectureList.each { architecture ->
 377     Views.ArchitectureViews[architecture] = listView(architecture) {
 378         recurse()
 379         columns {
 380             status()
 381             weather()
 382             name()
 383             lastSuccess()
 384             lastFailure()
 385             lastDuration()
 386             buildButton()
 387         }
 388     }
 389 }
 390
 391 // Create a view for non-PR jobs for each OS.
 392 Constants.osList.each { os ->
 393     // Don't create one for the special 'Windows_NT_BuildOnly'
 394     if (os == 'Windows_NT_BuildOnly') {
 395         return
 396     }
 397     Views.OSViews[os] = listView(os) {
 398         recurse()
 399         columns {
 400             status()
 401             weather()
 402             name()
 403             lastSuccess()
 404             lastFailure()
 405             lastDuration()
 406             buildButton()
 407         }
 408     }
 409 }
 410
 411 def static addToMergeView(def job) {
 412     Views.MergeJobView.with {
 413         jobs {
 414             name(job.name)
 415         }
 416     }
 417 }
 418
 419 def static addToPeriodicView(def job) {
 420     Views.PeriodicJobView.with {
 421         jobs {
 422             name(job.name)
 423         }
 424     }
 425 }
 426
 427 def static addToViews(def job, def isPR, def architecture, def os) {
 428     if (isPR) {
 429         // No views want PR jobs currently.
 430         return
 431     }
 432
 433     // Add to architecture view.
 434     Views.ArchitectureViews[architecture].with {
 435         jobs {
 436             name(job.name)
 437         }
 438     }
 439
 440     // Add to OS view.
 441     Views.OSViews[os].with {
 442         jobs {
 443             name(job.name)
 444         }
 445     }
 446 }
 447
 448 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 449     addToPeriodicView(job)
 450     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 451 }
 452
 453 def static addGithubPushTriggerHelper(def job) {
 454     addToMergeView(job)
 455     Utilities.addGithubPushTrigger(job)
 456 }
 457
 458
 459 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 460     assert os instanceof String
 461     assert architecture instanceof String
 462
 463     def armArches = ['arm', 'armlb', 'arm64']
 464     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 465
 466     if (!(architecture in armArches)) {
 467         assert options == null
 468         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 469
 470         return
 471     }
 472
 473     // This is an arm(64) job.
 474     //
 475     // There are several options.
 476     //
 477     // Windows_NT
 478     //
 479     // Arm32 (Build) -> latest-arm64
 480     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 481     // Arm32 (Test)  -> arm64-windows_nt
 482     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 483     //
 484     // Arm64 (Build) -> latest-arm64
 485     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 486     // Arm64 (Test)  -> arm64-windows_nt
 487     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 488     //
 489     // Ubuntu
 490     //
 491     // Arm32 (Build) -> arm-cross-latest
 492     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 493     // Arm32 (Test)  -> NYI Arch not supported
 494     //       |->
 495     //
 496     // Arm64 (Build) -> arm64-cross-latest
 497     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 498     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 499     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 500     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 501     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 502
 503     // This has to be a arm arch
 504     assert architecture in armArches
 505     if (os == "Windows_NT") {
 506         // Arm(64) Windows jobs share the same machines for now
 507         def isBuild = options['use_arm64_build_machine'] == true
 508
 509         if (isBuild == true) {
 510             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 511         } else {
 512             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 513         }
 514     } else {
 515         assert os != 'Windows_NT'
 516         assert os in supportedArmLinuxOs
 517
 518         if (architecture == 'arm' || architecture == 'armlb') {
 519             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 520         } else {
 521             // Arm64 Linux
 522             if (options['is_build_only'] == true) {
 523                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 524             } else {
 525                 // Arm64 Test Machines
 526                 if (options['large_pages'] == false) {
 527                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 528                 } else {
 529                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 530                 }
 531             }
 532         }
 533     }
 534 }
 535
 536 def static isGCStressRelatedTesting(def scenario) {
 537     // The 'r2r_gcstress15' scenario is a basic scenario.
 538     // Detect it and make it a GCStress related.
 539     if (scenario == 'r2r_gcstress15')
 540     {
 541         return true;
 542     }
 543
 544     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 545     def scenarioName = scenario.toLowerCase()
 546     def isGCStressTesting = false
 547     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 548         if (k in gcStressTestEnvVars) {
 549             isGCStressTesting = true;
 550         }
 551     }
 552     return isGCStressTesting
 553 }
 554
 555 def static isCoreFxScenario(def scenario) {
 556     def corefx_prefix = 'corefx_'
 557     if (scenario.length() < corefx_prefix.length()) {
 558         return false
 559     }
 560     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 561 }
 562
 563 def static isR2RBaselineScenario(def scenario) {
 564     return (scenario == 'r2r')
 565 }
 566
 567 def static isR2RStressScenario(def scenario) {
 568     return Constants.r2rStressScenarios.containsKey(scenario)
 569 }
 570
 571 def static isR2RScenario(def scenario) {
 572     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 573 }
 574
 575 def static isJitStressScenario(def scenario) {
 576     return Constants.jitStressModeScenarios.containsKey(scenario)
 577 }
 578
 579 def static isLongGc(def scenario) {
 580     return (scenario == 'longgc' || scenario == 'gcsimulator')
 581 }
 582
 583 def static isJitDiff(def scenario) {
 584     return (scenario == 'jitdiff')
 585 }
 586
 587 def static isGcReliabilityFramework(def scenario) {
 588     return (scenario == 'gc_reliability_framework')
 589 }
 590
 591 def static isArmWindowsScenario(def scenario) {
 592     return Constants.validArmWindowsScenarios.containsKey(scenario)
 593 }
 594
 595 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 596     if (isBuildOnly == true) {
 597         os = 'Windows_NT_buildOnly'
 598     }
 599
 600     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 601
 602     if (validOsPrTriggerArchConfigs == null) {
 603         return false
 604     }
 605
 606     if (validOsPrTriggerArchConfigs[architecture] != null) {
 607         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 608
 609         if (!(configuration in validOsPrTriggerConfigs)) {
 610             return false
 611         }
 612     } else {
 613         return false
 614     }
 615
 616     return true
 617 }
 618
 619 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 620     // 2 hours (120 minutes) is the default timeout
 621     def timeout = 120
 622     def innerLoop = (scenario == "innerloop")
 623
 624     if (!innerLoop) {
 625         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 626         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 627         timeout = 240
 628     }
 629
 630     if (!isBuildOnly) {
 631         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 632         if (isGCStressRelatedTesting(scenario)) {
 633             timeout = 4320
 634         }
 635         else if (isCoreFxScenario(scenario)) {
 636             timeout = 360
 637         }
 638         else if (isJitStressScenario(scenario)) {
 639             timeout = 240
 640         }
 641         else if (isR2RBaselineScenario(scenario)) {
 642             timeout = 240
 643         }
 644         else if (isLongGc(scenario)) {
 645             timeout = 1440
 646         }
 647         else if (isJitDiff(scenario)) {
 648             timeout = 240
 649         }
 650         else if (isGcReliabilityFramework(scenario)) {
 651             timeout = 1440
 652         }
 653         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 654             timeout = 240
 655         }
 656     }
 657
 658     if (configuration == 'Debug') {
 659         // Debug runs can be very slow. Add an hour.
 660         timeout += 60
 661     }
 662
 663     // If we've changed the timeout from the default, set it in the job.
 664
 665     if (timeout != 120) {
 666         Utilities.setJobTimeout(newJob, timeout)
 667     }
 668 }
 669
 670 def static getJobFolder(def scenario) {
 671     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 672         return 'jitstress'
 673     }
 674     if (scenario == 'illink') {
 675         return 'illink'
 676     }
 677     return ''
 678 }
 679
 680 def static getStressModeDisplayName(def scenario) {
 681     def displayStr = ''
 682     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 683         def prefixLength = 'COMPlus_'.length()
 684         if (k.length() >= prefixLength) {
 685             def modeName = k.substring(prefixLength, k.length())
 686             displayStr += ' ' + modeName + '=' + v
 687         }
 688     }
 689
 690     if (isCoreFxScenario(scenario)) {
 691         displayStr = ('CoreFx ' + displayStr).trim()
 692     }
 693
 694     return displayStr
 695 }
 696
 697 def static getR2RDisplayName(def scenario) {
 698     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 699     def displayStr = scenario
 700     def prefixLength = 'r2r_'.length()
 701     if (displayStr.length() >= prefixLength) {
 702         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 703     } else if (scenario == 'r2r') {
 704         displayStr = "R2R"
 705     }
 706     return displayStr
 707 }
 708
 709 //
 710 // Functions to create an environment script.
 711 //      envScriptCreate -- initialize the script (call first)
 712 //      envScriptFinalize -- finalize the script (call last)
 713 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 714 //      envScriptAppendExistingScript -- append an existing script to the generated script
 715 //
 716 // Each script returns a string of commands. Concatenate all the strings together before
 717 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 718 //
 719
 720 // Initialize the environment setting script.
 721 def static envScriptCreate(def os, def stepScriptLocation) {
 722     def stepScript = ''
 723     if (os == 'Windows_NT') {
 724         stepScript += "echo Creating TestEnv script\r\n"
 725         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 726
 727         // Create at least an empty script.
 728         stepScript += "echo. > ${stepScriptLocation}\r\n"
 729     }
 730     else {
 731         stepScript += "echo Creating environment setting script\n"
 732         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 733     }
 734
 735     return stepScript
 736 }
 737
 738 // Generates the string for setting stress mode variables.
 739 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 740     def stepScript = ''
 741     if (os == 'Windows_NT') {
 742         stressModeVars.each{ k, v ->
 743             // Write out what we are writing to the script file
 744             stepScript += "echo Setting ${k}=${v}\r\n"
 745             // Write out the set itself to the script file`
 746             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 747         }
 748     }
 749     else {
 750         stressModeVars.each{ k, v ->
 751             // Write out what we are writing to the script file
 752             stepScript += "echo Setting ${k}=${v}\n"
 753             // Write out the set itself to the script file`
 754             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 755         }
 756     }
 757
 758     return stepScript
 759 }
 760
 761 // Append an existing script to an environment script.
 762 // Returns string of commands to do this.
 763 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 764     assert (os == 'Windows_NT')
 765     def stepScript = ''
 766
 767     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 768     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 769
 770     return stepScript
 771 }
 772
 773 // Finalize an environment setting script.
 774 // Returns string of commands to do this.
 775 def static envScriptFinalize(def os, def stepScriptLocation) {
 776     def stepScript = ''
 777
 778     if (os == 'Windows_NT') {
 779         // Display the resulting script. This is useful when looking at the output log file.
 780         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 781         stepScript += "type ${stepScriptLocation}\r\n"
 782     }
 783     else {
 784         stepScript += "chmod +x ${stepScriptLocation}\n"
 785     }
 786
 787     return stepScript
 788 }
 789
 790 def static isNeedDocker(def architecture, def os, def isBuild) {
 791     if (isBuild) {
 792         if (architecture == 'x86' && os == 'Ubuntu') {
 793             return true
 794         }
 795         else if (architecture == 'arm') {
 796             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 797                 return true
 798             }
 799         }
 800     }
 801     else {
 802         if (architecture == 'x86' && os == 'Ubuntu') {
 803             return true
 804         }
 805     }
 806     return false
 807 }
 808
 809 def static getDockerImageName(def architecture, def os, def isBuild) {
 810     // We must change some docker private images to official later
 811     if (isBuild) {
 812         if (architecture == 'x86' && os == 'Ubuntu') {
 813             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 814         }
 815         else if (architecture == 'arm') {
 816             if (os == 'Ubuntu') {
 817                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 818             }
 819             else if (os == 'Ubuntu16.04') {
 820                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 821             }
 822             else if (os == 'Tizen') {
 823                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 824             }
 825         }
 826     }
 827     else {
 828         if (architecture == 'x86' && os == 'Ubuntu') {
 829             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 830         }
 831     }
 832     println("Unknown architecture to use docker: ${architecture} ${os}");
 833     assert false
 834 }
 835
 836 // Calculates the name of the build job based on some typical parameters.
 837 //
 838 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 839     // If the architecture is x64, do not add that info into the build name.
 840     // Need to change around some systems and other builds to pick up the right builds
 841     // to do that.
 842
 843     def suffix = scenario != 'normal' ? "_${scenario}" : '';
 844     if (isBuildOnly) {
 845         suffix += '_bld'
 846     }
 847     def baseName = ''
 848     switch (architecture) {
 849         case 'x64':
 850             if (scenario == 'normal') {
 851                 // For now we leave x64 off of the name for compatibility with other jobs
 852                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 853             }
 854             else if (scenario == 'formatting') {
 855                 // we don't care about the configuration for the formatting job. It runs all configs
 856                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 857             }
 858             else {
 859                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 860             }
 861             break
 862         case 'arm64':
 863             if (os.toLowerCase() == "windows_nt") {
 864                 // These are cross builds
 865                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 866             }
 867             else {
 868                 // Defaults to a small page size set of machines.
 869                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 870             }
 871             break
 872         case 'arm':
 873             // These are cross builds
 874             if (os == 'Tizen') {
 875                 // ABI: softfp
 876                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 877             }
 878             else {
 879                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 880             }
 881             break
 882         case 'armlb':
 883             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 884             break
 885         case 'x86':
 886         case 'x86_arm_altjit':
 887         case 'x64_arm64_altjit':
 888             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 889             break
 890         default:
 891             println("Unknown architecture: ${architecture}");
 892             assert false
 893             break
 894     }
 895
 896     return baseName + suffix
 897 }
 898
 899 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 900
 901     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 902     if (os == 'Windows_NT') {
 903         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 904             if (!(branch in Constants.WindowsArm64Branches)) {
 905                 return
 906             }
 907         }
 908     }
 909
 910     // Check scenario.
 911     switch (scenario) {
 912         case 'innerloop':
 913             break
 914         case 'normal':
 915             switch (architecture) {
 916                 case 'x64':
 917                 case 'x86':
 918                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 919                         addPeriodicTriggerHelper(job, '@daily')
 920                     }
 921                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 922                         addGithubPushTriggerHelper(job)
 923                     }
 924                     break
 925                 case 'arm':
 926                 case 'armlb':
 927                 case 'x86_arm_altjit':
 928                 case 'x64_arm64_altjit':
 929                     addGithubPushTriggerHelper(job)
 930                     break
 931                 case 'arm64':
 932                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 933                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 934                     break
 935                 default:
 936                     println("Unknown architecture: ${architecture}");
 937                     assert false
 938                     break
 939             }
 940             break
 941         case 'r2r':
 942             assert !(os in bidailyCrossList)
 943             // r2r gets a push trigger for checked/release
 944             if (configuration == 'Checked' || configuration == 'Release') {
 945                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 946                 if (architecture == 'x64' && os != 'OSX10.12') {
 947                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 948                     if (isFlowJob || os == 'Windows_NT') {
 949                         addGithubPushTriggerHelper(job)
 950                     }
 951                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 952                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 953                     if (isFlowJob) {
 954                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 955                     }
 956                 }
 957                 // For x86, only add per-commit jobs for Windows
 958                 else if (architecture == 'x86') {
 959                     if (os == 'Windows_NT') {
 960                         addGithubPushTriggerHelper(job)
 961                     }
 962                 }
 963                 // arm64 r2r jobs should only run daily.
 964                 else if (architecture == 'arm64') {
 965                     if (os == 'Windows_NT') {
 966                         addPeriodicTriggerHelper(job, '@daily')
 967                     }
 968                 }
 969             }
 970             break
 971         case 'r2r_jitstress1':
 972         case 'r2r_jitstress2':
 973         case 'r2r_jitstressregs1':
 974         case 'r2r_jitstressregs2':
 975         case 'r2r_jitstressregs3':
 976         case 'r2r_jitstressregs4':
 977         case 'r2r_jitstressregs8':
 978         case 'r2r_jitstressregs0x10':
 979         case 'r2r_jitstressregs0x80':
 980         case 'r2r_jitstressregs0x1000':
 981         case 'r2r_jitminopts':
 982         case 'r2r_jitforcerelocs':
 983         case 'r2r_gcstress15':
 984             assert !(os in bidailyCrossList)
 985
 986             // GCStress=C is currently not supported on OS X
 987             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 988                 break
 989             }
 990
 991             // GC Stress 15 r2r gets a push trigger for checked/release
 992             if (configuration == 'Checked' || configuration == 'Release') {
 993                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 994                 if (architecture == 'x64') {
 995                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 996                     if (isFlowJob || os == 'Windows_NT') {
 997                         // Add a weekly periodic trigger
 998                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 999                     }
1000                 }
1001                 // For x86, only add per-commit jobs for Windows
1002                 else if (architecture == 'x86') {
1003                     if (os == 'Windows_NT') {
1004                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1005                     }
1006                 }
1007             }
1008             break
1009         case 'longgc':
1010             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1011             assert configuration == 'Release'
1012             assert architecture == 'x64'
1013             addPeriodicTriggerHelper(job, '@daily')
1014             // TODO: Add once external email sending is available again
1015             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1016             break
1017         case 'gcsimulator':
1018             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1019             assert configuration == 'Release'
1020             assert architecture == 'x64'
1021             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1022             // TODO: Add once external email sending is available again
1023             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1024             break
1025         case 'standalone_gc':
1026             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1027             assert (configuration == 'Release' || configuration == 'Checked')
1028             // TODO: Add once external email sending is available again
1029             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1030             addPeriodicTriggerHelper(job, '@daily')
1031             break
1032         case 'gc_reliability_framework':
1033             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1034             assert (configuration == 'Release' || configuration == 'Checked')
1035             // Only triggered by phrase.
1036             break
1037         case 'ilrt':
1038             assert !(os in bidailyCrossList)
1039             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1040             if (architecture == 'x64' && configuration == 'Release') {
1041                 // We don't expect to see a job generated except in these scenarios
1042                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1043                 if (isFlowJob || os == 'Windows_NT') {
1044                     addPeriodicTriggerHelper(job, '@daily')
1045                 }
1046             }
1047             break
1048         case 'jitdiff':
1049             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1050             assert configuration == 'Checked'
1051             assert (architecture == 'x64' || architecture == 'x86')
1052             addGithubPushTriggerHelper(job)
1053             break
1054         case 'formatting':
1055             assert (os == 'Windows_NT' || os == "Ubuntu")
1056             assert architecture == 'x64'
1057             addGithubPushTriggerHelper(job)
1058             break
1059         case 'jitstressregs1':
1060         case 'jitstressregs2':
1061         case 'jitstressregs3':
1062         case 'jitstressregs4':
1063         case 'jitstressregs8':
1064         case 'jitstressregs0x10':
1065         case 'jitstressregs0x80':
1066         case 'jitstressregs0x1000':
1067         case 'minopts':
1068         case 'forcerelocs':
1069         case 'jitstress1':
1070         case 'jitstress2':
1071         case 'jitstress2_jitstressregs1':
1072         case 'jitstress2_jitstressregs2':
1073         case 'jitstress2_jitstressregs3':
1074         case 'jitstress2_jitstressregs4':
1075         case 'jitstress2_jitstressregs8':
1076         case 'jitstress2_jitstressregs0x10':
1077         case 'jitstress2_jitstressregs0x80':
1078         case 'jitstress2_jitstressregs0x1000':
1079         case 'tailcallstress':
1080         case 'jitsse2only':
1081         case 'jitnosimd':
1082         case 'jitnox86hwintrinsic':
1083         case 'jitincompletehwintrinsic':
1084         case 'jitx86hwintrinsicnoavx':
1085         case 'jitx86hwintrinsicnoavx2':
1086         case 'jitx86hwintrinsicnosimd':
1087         case 'corefx_baseline':
1088         case 'corefx_minopts':
1089         case 'corefx_jitstress1':
1090         case 'corefx_jitstress2':
1091         case 'corefx_jitstressregs1':
1092         case 'corefx_jitstressregs2':
1093         case 'corefx_jitstressregs3':
1094         case 'corefx_jitstressregs4':
1095         case 'corefx_jitstressregs8':
1096         case 'corefx_jitstressregs0x10':
1097         case 'corefx_jitstressregs0x80':
1098         case 'corefx_jitstressregs0x1000':
1099         case 'zapdisable':
1100             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1101                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1102                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1103                     if (os == 'Windows_NT') {
1104                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1105                         addPeriodicTriggerHelper(job, '@weekly')
1106                     }
1107                 }
1108                 else {
1109                     addPeriodicTriggerHelper(job, '@daily')
1110                 }
1111             }
1112             break
1113         case 'heapverify1':
1114         case 'gcstress0x3':
1115             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1116                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1117                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1118                     if (os == 'Windows_NT') {
1119                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1120                         addPeriodicTriggerHelper(job, '@weekly')
1121                     }
1122                     // TODO: Add once external email sending is available again
1123                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1124                 }
1125                 else {
1126                     addPeriodicTriggerHelper(job, '@weekly')
1127                 }
1128             }
1129             break
1130         case 'gcstress0xc':
1131         case 'gcstress0xc_zapdisable':
1132         case 'gcstress0xc_zapdisable_jitstress2':
1133         case 'gcstress0xc_zapdisable_heapverify1':
1134         case 'gcstress0xc_jitstress1':
1135         case 'gcstress0xc_jitstress2':
1136         case 'gcstress0xc_minopts_heapverify1':
1137             // GCStress=C is currently not supported on OS X
1138             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1139                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1140                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1141                     if (os == 'Windows_NT') {
1142                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1143                         addPeriodicTriggerHelper(job, '@weekly')
1144                     }
1145                     // TODO: Add once external email sending is available again
1146                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1147                 }
1148                 else {
1149                     addPeriodicTriggerHelper(job, '@weekly')
1150                 }
1151             }
1152             break
1153
1154         case 'illink':
1155             // Testing on other operating systems TBD
1156             assert (os == 'Windows_NT' || os == 'Ubuntu')
1157             if (architecture == 'x64' || architecture == 'x86') {
1158                 if (configuration == 'Checked') {
1159                     addPeriodicTriggerHelper(job, '@daily')
1160                 }
1161             }
1162             break
1163
1164         case 'tieredcompilation':
1165         case 'corefx_tieredcompilation':
1166             // No periodic jobs just yet, still testing
1167             break
1168
1169         default:
1170             println("Unknown scenario: ${scenario}");
1171             assert false
1172             break
1173     }
1174     return
1175 }
1176
1177 // **************************
1178 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1179 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1180 // builds will do a couple extra steps.
1181 // **************************
1182
1183 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1184 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1185 // it's just used for internal builds
1186 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1187 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1188     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1189
1190     if (isWindowsBuildOnlyJob) {
1191         return
1192     }
1193
1194     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1195     // Non pull request builds.
1196     if (!isPR) {
1197         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1198         return
1199     }
1200
1201      def arm64Users = [
1202         'AndyAyersMS',
1203         'briansull',
1204         'BruceForstall',
1205         'CarolEidt',
1206         'cmckinsey',
1207         'erozenfeld',
1208         'janvorli',
1209         'jashook',
1210         'JosephTremoulet',
1211         'pgodeq',
1212         'pgavlin',
1213         'rartemev',
1214         'russellhadley',
1215         'RussKeldorph',
1216         'sandreenko',
1217         'sdmaclea',
1218         'swaroop-sridhar',
1219         'jkotas',
1220         'markwilkie',
1221         'weshaggard'
1222     ]
1223
1224     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1225     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1226     def osGroup = getOSGroup(os)
1227     switch (architecture) {
1228         case 'x64': // editor brace matching: {
1229             if (scenario == 'formatting') {
1230                 assert configuration == 'Checked'
1231                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1232                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1233                 }
1234
1235                 break
1236             }
1237
1238             switch (os) {
1239                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1240                 case 'Debian8.4':
1241                 case 'RHEL7.2':
1242                     if (scenario == 'innerloop') {
1243                         assert !isFlowJob
1244                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1245                     }
1246                     else if (scenario == 'normal') {
1247                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1248                     }
1249                     break
1250
1251                 case 'Ubuntu16.04':
1252                     assert !isFlowJob
1253                     assert scenario != 'innerloop'
1254                     // Distinguish with the other architectures (arm and x86)
1255                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1256                     break
1257
1258                 case 'Fedora24':
1259                 case 'Ubuntu16.10':
1260                     assert !isFlowJob
1261                     assert scenario != 'innerloop'
1262                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1263                     break
1264
1265                 case 'Ubuntu':
1266                     if (scenario == 'illink') {
1267                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1268                         break
1269                     }
1270                     // fall through
1271
1272                 case 'OSX10.12':
1273                     // Triggers on the non-flow jobs aren't necessary here
1274                     // Corefx testing uses non-flow jobs.
1275                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1276                         break
1277                     }
1278                     switch (scenario) {
1279                         case 'innerloop':
1280                             // PR Triggered jobs. These jobs will run pri0 tests.
1281                             if (configuration == 'Checked') {
1282                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test.*")
1283                             }
1284                             break
1285
1286                         case 'normal':
1287                             // OSX uses checked for default PR tests
1288                             if (configuration == 'Checked') {
1289                                 // Default trigger
1290                                 assert !job.name.contains("centos")
1291                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1292                             }
1293                             break
1294
1295                         case 'jitdiff':
1296                             if (configuration == 'Checked') {
1297                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1298                             }
1299                             break
1300
1301                         case 'ilrt':
1302                             if (configuration == 'Release') {
1303                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1304                             }
1305                             break
1306
1307                         case 'longgc':
1308                             if (configuration == 'Release') {
1309                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1310                             }
1311                             break
1312
1313                         case 'gcsimulator':
1314                             if (configuration == 'Release') {
1315                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1316                             }
1317                             break
1318
1319                         case 'standalone_gc':
1320                             if (configuration == 'Release' || configuration == 'Checked') {
1321                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1322                             }
1323                             break
1324
1325                         case 'gc_reliability_framework':
1326                             if (configuration == 'Release' || configuration == 'Checked') {
1327                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1328                             }
1329                             break
1330
1331                         default:
1332                             if (isJitStressScenario(scenario)) {
1333                                 def displayStr = getStressModeDisplayName(scenario)
1334                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1335                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1336                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1337                             }
1338                             else if (isR2RScenario(scenario)) {
1339                                 if (configuration == 'Release' || configuration == 'Checked') {
1340                                     def displayStr = getR2RDisplayName(scenario)
1341                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1342                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1343                                 }
1344                             }
1345                             else {
1346                                 println("Unknown scenario: ${scenario}");
1347                                 assert false
1348                             }
1349                             break
1350
1351                     }
1352                     break
1353
1354                 case 'CentOS7.1':
1355                     switch (scenario) {
1356                         case 'innerloop':
1357                             // CentOS uses checked for default PR tests while debug is build only
1358                             if (configuration == 'Debug') {
1359                                 // Default trigger
1360                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1361                             }
1362
1363                             // Make sure this is a flow job to get build and test.
1364                             if (configuration == 'Checked' && isFlowJob) {
1365                                 assert job.name.contains("flow")
1366                                 // Default trigger
1367                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1368                             }
1369                             break
1370
1371                         case 'normal':
1372                             // Make sure this is a flow job to get build and test.
1373                             if (configuration == 'Checked' && isFlowJob) {
1374                                 assert job.name.contains("flow")
1375                                 // Default trigger
1376                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1377                             }
1378                             break
1379
1380                         default:
1381                             if (isR2RScenario(scenario)) {
1382                                 if (configuration == 'Release' || configuration == 'Checked') {
1383                                     def displayStr = getR2RDisplayName(scenario)
1384                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1385                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1386                                 }
1387                             }
1388                             break
1389
1390                     }
1391                     break
1392
1393                 case 'Windows_NT':
1394                     switch (scenario) {
1395                         case 'innerloop':
1396                             // Default trigger
1397                             if (configuration == 'Checked' || configuration == 'Release') {
1398                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1399                             }
1400                             break
1401
1402                         case 'normal':
1403                             if (configuration == 'Checked') {
1404                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1405                             }
1406                             break
1407
1408                         case 'jitdiff':
1409                             if (configuration == 'Checked') {
1410                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1411                             }
1412                             break
1413
1414                         case 'ilrt':
1415                             if (configuration == 'Release') {
1416                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1417                             }
1418                             break
1419
1420                         case 'longgc':
1421                             if (configuration == 'Release') {
1422                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1423                             }
1424                             break
1425
1426                         case 'gcsimulator':
1427                             if (configuration == 'Release') {
1428                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1429                             }
1430                             break
1431
1432                         case 'standalone_gc':
1433                             if (configuration == 'Release' || configuration == 'Checked') {
1434                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1435                             }
1436                             break
1437
1438                         case 'gc_reliability_framework':
1439                             if (configuration == 'Release' || configuration == 'Checked') {
1440                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1441                             }
1442                             break
1443
1444                         case 'illink':
1445                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1446                             break
1447
1448                         default:
1449                             if (isJitStressScenario(scenario)) {
1450                                 def displayStr = getStressModeDisplayName(scenario)
1451                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1452                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1453                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1454                             }
1455                             else if (isR2RScenario(scenario)) {
1456                                 if (configuration == 'Release' || configuration == 'Checked') {
1457                                     def displayStr = getR2RDisplayName(scenario)
1458                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1459                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1460                                 }
1461                             }
1462                             else {
1463                                 println("Unknown scenario: ${scenario}");
1464                                 assert false
1465                             }
1466                             break
1467
1468                     }
1469                     break
1470
1471                 default:
1472                     println("Unknown os: ${os}");
1473                     assert false
1474                     break
1475
1476             }
1477
1478             break
1479
1480         // editor brace matching: }
1481         case 'armlb':
1482         case 'arm': // editor brace matching: {
1483             switch (os) {
1484                 case 'Ubuntu':
1485                 case 'Ubuntu16.04':
1486                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1487                         break
1488                     }
1489
1490                     assert scenario != 'innerloop'
1491                     job.with {
1492                         publishers {
1493                             azureVMAgentPostBuildAction {
1494                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1495                             }
1496                         }
1497                     }
1498                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1499                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1500                     break
1501
1502                 case 'Tizen':
1503                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1504                         break
1505                     }
1506
1507                     architecture = 'armel'
1508                     job.with {
1509                         publishers {
1510                             azureVMAgentPostBuildAction {
1511                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1512                             }
1513                         }
1514                     }
1515
1516                     if (scenario == 'innerloop') {
1517                         if (configuration == 'Checked') {
1518                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1519                         }
1520                     }
1521                     else {
1522                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1523                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1524                     }
1525                     break
1526
1527                 case 'Windows_NT':
1528                     // Triggers on the non-flow jobs aren't necessary here
1529                     if (!isFlowJob) {
1530                         break
1531                     }
1532
1533                     // Set up a private trigger
1534                     def contextString = "${os} ${architecture} Cross ${configuration}"
1535                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1536                     if (scenario == 'innerloop') {
1537                         contextString += " Innerloop"
1538                         triggerString += "\\W+Innerloop"
1539                     }
1540                     else {
1541                         contextString += " ${scenario}"
1542                         triggerString += "\\W+${scenario}"
1543                     }
1544
1545                     if (configuration == 'Debug') {
1546                         contextString += " Build"
1547                         triggerString += "\\W+Build"
1548                     } else {
1549                         contextString += " Build and Test"
1550                         triggerString += "\\W+Build and Test"
1551                     }
1552
1553                     triggerString += ".*"
1554
1555                     switch (scenario) {
1556                         case 'innerloop':
1557                             // Only Checked is an innerloop trigger.
1558                             if (configuration == 'Checked')
1559                             {
1560                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1561                             }
1562                             break
1563                         case 'normal':
1564                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1565                             break
1566                         default:
1567                             // Stress jobs will use this code path.
1568                             if (isArmWindowsScenario(scenario)) {
1569                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1570                             }
1571                             break
1572                     }
1573                     break
1574                 default:
1575                     println("NYI os: ${os}");
1576                     assert false
1577                     break
1578             }
1579             break
1580         // editor brace matching: }
1581         case 'arm64': // editor brace matching: {
1582             // Set up a private trigger
1583             def contextString = "${os} ${architecture} Cross ${configuration}"
1584             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1585
1586             if (scenario == 'innerloop') {
1587                 contextString += " Innerloop"
1588                 triggerString += "\\W+Innerloop"
1589             }
1590             else {
1591                 contextString += " ${scenario}"
1592                 triggerString += "\\W+${scenario}"
1593             }
1594
1595             if (configuration == 'Debug') {
1596                 contextString += " Build"
1597                 triggerString += "\\W+Build"
1598             } else {
1599                 contextString += " Build and Test"
1600                 triggerString += "\\W+Build and Test"
1601             }
1602
1603             triggerString += ".*"
1604
1605             switch (os) {
1606                 case 'Ubuntu':
1607                 case 'Ubuntu16.04':
1608                     switch (scenario) {
1609                         case 'innerloop':
1610                             if (configuration == 'Debug' && !isFlowJob) {
1611                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1612                             }
1613
1614                             break
1615                         case 'normal':
1616                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1617                             break
1618                         default:
1619                             if (isR2RScenario(scenario)) {
1620                                 if (configuration == 'Checked' || configuration == 'Release') {
1621                                     def displayStr = getR2RDisplayName(scenario)
1622                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1623                                 }
1624                             }
1625                             break
1626                     }
1627                     break
1628
1629                 case 'Windows_NT':
1630                     // Triggers on the non-flow jobs aren't necessary here
1631                     if (!isFlowJob) {
1632                         break
1633                     }
1634
1635                     assert isArmWindowsScenario(scenario)
1636                     switch (scenario) {
1637                         case 'innerloop':
1638                             if (configuration == 'Checked') {
1639                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1640                             }
1641
1642                             break
1643                         case 'normal':
1644                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1645                             break
1646                         default:
1647                             // Stress jobs will use this code path.
1648                             if (isArmWindowsScenario(scenario)) {
1649                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1650                             }
1651                             break
1652                     }
1653                     break
1654                 default:
1655                     println("NYI os: ${os}");
1656                     assert false
1657                     break
1658             }
1659             break
1660
1661         // editor brace matching: }
1662         case 'x86': // editor brace matching: {
1663             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1664             if (os == 'Ubuntu') {
1665                 // Triggers on the non-flow jobs aren't necessary here
1666                 if (!isFlowJob) {
1667                     break
1668                 }
1669
1670                 // on-demand only for ubuntu x86
1671                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1672                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1673                 break
1674
1675             }
1676             switch (scenario) {
1677                 case 'innerloop':
1678                     if (configuration == 'Checked' || configuration == 'Release') {
1679                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1680                     }
1681                     break
1682
1683                 case 'normal':
1684                     if (configuration == 'Checked') {
1685                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1686                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1687                     }
1688                     break
1689
1690                 case 'ilrt':
1691                     if (configuration == 'Release') {
1692                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1693                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1694                     }
1695                     break
1696
1697                 case 'longgc':
1698                     if (configuration == 'Release') {
1699                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1700                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1701                     }
1702                     break
1703
1704                 case 'gcsimulator':
1705                     if (configuration == 'Release') {
1706                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1707                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1708                     }
1709                     break
1710
1711                 case 'standalone_gc':
1712                     if (configuration == 'Release' || configuration == 'Checked') {
1713                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1714                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1715                     }
1716                     break
1717
1718                 case 'illink':
1719                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1720                     break
1721
1722                 default:
1723                     if (isJitStressScenario(scenario)) {
1724                         def displayStr = getStressModeDisplayName(scenario)
1725                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1726                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1727                     }
1728                     else if (isR2RScenario(scenario)) {
1729                         if (configuration == 'Release' || configuration == 'Checked') {
1730                             def displayStr = getR2RDisplayName(scenario)
1731                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1732                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1733                         }
1734                     }
1735                     else {
1736                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1737                         assert false
1738                     }
1739                     break
1740
1741             }
1742             break
1743
1744          // editor brace matching: }
1745         case 'x64_arm64_altjit':
1746         case 'x86_arm_altjit': // editor brace matching: {
1747             assert (os == 'Windows_NT')
1748             switch (scenario) {
1749                 case 'normal':
1750                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1751                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1752                     break
1753                 default:
1754                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1755                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1756                     break
1757             }
1758             break
1759
1760         // editor brace matching: }
1761         default:
1762             println("Unknown architecture: ${architecture}");
1763             assert false
1764             break
1765     }
1766 }
1767
1768 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1769     def buildCommands = [];
1770     def osGroup = getOSGroup(os)
1771     def lowerConfiguration = configuration.toLowerCase()
1772
1773     def priority = '1'
1774     if (scenario == 'innerloop') {
1775         priority = '0'
1776     }
1777
1778     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1779
1780     def enableCorefxTesting = isCoreFxScenario(scenario)
1781
1782     // Calculate the build steps, archival, and xunit results
1783     switch (os) {
1784         case 'Windows_NT': // editor brace matching: {
1785             switch (architecture) {
1786                 case 'x64':
1787                 case 'x86':
1788                 case 'x86_arm_altjit':
1789                 case 'x64_arm64_altjit':
1790                     def arch = architecture
1791                     def buildOpts = ''
1792                     if (architecture == 'x86_arm_altjit') {
1793                         arch = 'x86'
1794                     }
1795                     else if (architecture == 'x64_arm64_altjit') {
1796                         arch = 'x64'
1797                     }
1798
1799                     if (scenario == 'formatting') {
1800                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1801                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1802                         break
1803                     }
1804
1805                     if (scenario == 'illink') {
1806                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1807                     }
1808
1809                     // If it is a release build for windows, ensure PGO is used, else fail the build
1810                     if ((lowerConfiguration == 'release') &&
1811                         (scenario in Constants.basicScenarios) &&
1812                         (architecture != 'x86_arm_altjit') &&
1813                         (architecture != 'x64_arm64_altjit')) {
1814
1815                         buildOpts += ' -enforcepgo'
1816                     }
1817
1818                     if (enableCorefxTesting) {
1819                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
1820                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
1821                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
1822                         //
1823                         // buildOpts += ' skiptests';
1824                         buildOpts += " -priority=0"
1825                     } else {
1826                         buildOpts += " -priority=${priority}"
1827                     }
1828
1829                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1830                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1831                     // 35 characters long.
1832
1833                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1834
1835                     if (!isBuildOnly) {
1836                         def runtestArguments = ''
1837                         def testOpts = 'collectdumps'
1838
1839                         if (isR2RScenario(scenario)) {
1840
1841                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1842                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1843                             // to cause the tests to be crossgen'ed.
1844
1845                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1846                                 testOpts += ' crossgenaltjit protononjit.dll'
1847                             } else {
1848                                 testOpts += ' crossgen'
1849                             }
1850
1851                             testOpts += ' runcrossgentests'
1852                         }
1853                         else if (scenario == 'jitdiff') {
1854                             testOpts += ' jitdisasm crossgen'
1855                         }
1856                         else if (scenario == 'ilrt') {
1857                             testOpts += ' ilasmroundtrip'
1858                         }
1859                         else if (isLongGc(scenario)) {
1860                             testOpts += " ${scenario} sequential"
1861                         }
1862                         else if (scenario == 'standalone_gc') {
1863                             testOpts += ' gcname clrgc.dll'
1864                         }
1865                         else if (scenario == 'illink') {
1866                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1867                         }
1868
1869                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1870                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1871                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1872                         // which is read by the xunit harness.)
1873                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1874                         {
1875                             def timeout = 1800000
1876                             testOpts += " timeout ${timeout}"
1877                         }
1878
1879                         // If we are running a stress mode, we should write out the set of key
1880                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1881
1882                         def envScriptPath = ''
1883                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1884                             def buildCommandsStr = ''
1885                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1886                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1887
1888                             if (isJitStressScenario(scenario)) {
1889                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1890                             }
1891                             else if (isR2RStressScenario(scenario)) {
1892                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1893                             }
1894
1895                             if (architecture == 'x86_arm_altjit') {
1896                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1897                             }
1898                             else if (architecture == 'x64_arm64_altjit') {
1899                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1900                             }
1901
1902                             envScriptFinalize(os, envScriptPath)
1903
1904                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1905                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1906                             // into a single script.
1907                             buildCommands += buildCommandsStr
1908                         }
1909                         else if (architecture == 'x86_arm_altjit') {
1910                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1911                         }
1912                         else if (architecture == 'x64_arm64_altjit') {
1913                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1914                         }
1915                         if (envScriptPath != '') {
1916                             testOpts += " TestEnv ${envScriptPath}"
1917                         }
1918
1919                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1920
1921                         if (enableCorefxTesting) {
1922                             // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
1923                             // to determine the correct matching corefx version git commit hash.
1924                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1925
1926                             def workspaceRelativeFxRoot = "_/fx"
1927                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1928
1929                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1930
1931                             // Archive and process (only) the test results
1932                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1933                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1934
1935                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1936                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1937                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1938                         }
1939                         else if (isGcReliabilityFramework(scenario)) {
1940                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1941                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1942                         }
1943                         else {
1944                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1945                         }
1946                     }
1947
1948                     if (!enableCorefxTesting) {
1949                         // Run the rest of the build
1950                         // Build the mscorlib for the other OS's
1951                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1952                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1953
1954                         if (arch == "x64") {
1955                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1956                         }
1957
1958                         // Zip up the tests directory so that we don't use so much space/time copying
1959                         // 10s of thousands of files around.
1960                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1961
1962                         if (!isJitStressScenario(scenario)) {
1963                             // For windows, pull full test results and test drops for x86/x64.
1964                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1965                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1966                         }
1967
1968                         if (scenario == 'jitdiff') {
1969                             // retrive jit-dasm output for base commit, and run jit-diff
1970                             if (!isBuildOnly) {
1971                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1972                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1973                             }
1974                         }
1975
1976                         if (!isBuildOnly) {
1977                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1978                         }
1979                     }
1980                     break
1981                 case 'armlb':
1982                 case 'arm':
1983                     assert isArmWindowsScenario(scenario)
1984
1985                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1986                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1987
1988                     def buildArchitecture = 'arm'
1989
1990                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1991                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1992                     def armCrossgenOpt = ''
1993                     if (architecture == 'armlb') {
1994                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1995                     }
1996
1997                     // This is now a build only job. Do not run tests. Use the flow job.
1998                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1999
2000                     // Zip up the tests directory so that we don't use so much space/time copying
2001                     // 10s of thousands of files around.
2002                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2003
2004                     // Add archival.
2005                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2006                     break
2007                 case 'arm64':
2008                     assert isArmWindowsScenario(scenario)
2009
2010                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
2011                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2012
2013                     // This is now a build only job. Do not run tests. Use the flow job.
2014                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2015
2016                     // Zip up the tests directory so that we don't use so much space/time copying
2017                     // 10s of thousands of files around.
2018                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2019
2020                     // Add archival.
2021                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2022                     break
2023                 default:
2024                     println("Unknown architecture: ${architecture}");
2025                     assert false
2026                     break
2027             }
2028             break
2029         // editor brace matching: }
2030         case 'Ubuntu':
2031         case 'Ubuntu16.04':
2032         case 'Ubuntu16.10':
2033         case 'Debian8.4':
2034         case 'OSX10.12':
2035         case 'CentOS7.1':
2036         case 'RHEL7.2':
2037         case 'Tizen':
2038         case 'Fedora24': // editor brace matching: {
2039             switch (architecture) {
2040                 case 'x64':
2041                 case 'x86':
2042                     if (architecture == 'x86' && os == 'Ubuntu') {
2043                         // build and PAL test
2044                         def dockerImage = getDockerImageName(architecture, os, true)
2045                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2046                         dockerImage = getDockerImageName(architecture, os, false)
2047                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2048                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2049                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2050                         break
2051                     }
2052
2053                     if (scenario == 'formatting') {
2054                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2055                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2056                         break
2057                     }
2058
2059                     if (scenario == 'illink') {
2060                         assert(os == 'Ubuntu')
2061                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2062                     }
2063
2064                     if (!enableCorefxTesting) {
2065                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2066                         // only on supported OS platforms.
2067                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2068                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2069
2070                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
2071                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2072
2073                         // Basic archiving of the build
2074                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2075                         // And pal tests
2076                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2077                     }
2078                     else {
2079                         // Corefx stress testing
2080                         assert os == 'Ubuntu'
2081                         assert architecture == 'x64'
2082                         assert lowerConfiguration == 'checked'
2083                         assert isJitStressScenario(scenario)
2084
2085                         // Build coreclr
2086                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2087
2088                         // Generate the test layout, which restores the corefx package, and allows run-corefx-tests.py to determine the
2089                         // matching corefx version git commit hash.
2090                         buildCommands += "./build-test.sh ${architecture} ${lowerConfiguration} generatelayoutonly"
2091
2092                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2093
2094                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2095                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2096                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2097                         buildCommands += envScriptCmds
2098
2099                         // Build and text corefx
2100                         def workspaceRelativeFxRoot = "_/fx"
2101                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2102
2103                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
2104
2105                         // Archive and process (only) the test results
2106                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2107                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2108                     }
2109                     break
2110                 case 'arm64':
2111                     if (!enableCorefxTesting) {
2112                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
2113
2114                         // HACK -- Arm64 does not have corefx jobs yet.
2115                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2116                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
2117                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2118                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
2119
2120                         // Basic archiving of the build
2121                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2122                     }
2123                     break
2124                 case 'arm':
2125                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2126                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2127
2128                     // default values for Ubuntu
2129                     def arm_abi="arm"
2130                     def linuxCodeName="trusty"
2131                     if (os == 'Ubuntu16.04') {
2132                         linuxCodeName="xenial"
2133                     }
2134                     else if (os == 'Tizen') {
2135                         arm_abi="armel"
2136                         linuxCodeName="tizen"
2137                     }
2138
2139                     // Unzip the Windows test binaries first. Exit with 0
2140                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2141
2142                     // Unpack the corefx binaries
2143                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2144                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2145                     if (os != 'Tizen') {
2146                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2147                     }
2148                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2149                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2150                     if (os == 'Ubuntu') {
2151                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2152                     }
2153
2154                     // Call the ARM CI script to cross build and test using docker
2155                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2156                     --mode=docker \\
2157                     --${arm_abi} \\
2158                     --linuxCodeName=${linuxCodeName} \\
2159                     --buildConfig=${lowerConfiguration} \\
2160                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2161                     --coreFxBinDir=./bin/CoreFxBinDir \\
2162                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2163
2164                     // Basic archiving of the build, no pal tests
2165                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2166                     break
2167                 default:
2168                     println("Unknown architecture: ${architecture}");
2169                     assert false
2170                     break
2171             }
2172             break
2173         // editor brace matching: }
2174         default:
2175             println("Unknown os: ${os}");
2176             assert false
2177             break
2178     } // os
2179
2180     return buildCommands
2181 }
2182
2183 Constants.allScenarios.each { scenario ->
2184     [true, false].each { isPR ->
2185         Constants.architectureList.each { architecture ->
2186             Constants.configurationList.each { configuration ->
2187                 Constants.osList.each { os ->
2188                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2189                     // and reset the os to Windows_NT
2190                     def isBuildOnly = false
2191                     if (os == 'Windows_NT_BuildOnly') {
2192                         isBuildOnly = true
2193                         os = 'Windows_NT'
2194                     }
2195
2196                     // Tizen is only supported for arm architecture
2197                     if (os == 'Tizen' && architecture != 'arm') {
2198                         return
2199                     }
2200
2201                     // Skip totally unimplemented (in CI) configurations.
2202                     switch (architecture) {
2203                         case 'arm64':
2204                             if (os == 'Ubuntu16.04') {
2205                                 os = 'Ubuntu'
2206                             }
2207
2208                             // Windows and Ubuntu only
2209                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2210                                 return
2211                             }
2212                             break
2213                         case 'arm':
2214                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2215                                 return
2216                             }
2217                             break
2218                         case 'armlb':
2219                             if (os != 'Windows_NT') {
2220                                 return
2221                             }
2222                             break
2223                         case 'x86':
2224                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2225                                 return
2226                             }
2227                             break
2228                         case 'x86_arm_altjit':
2229                         case 'x64_arm64_altjit':
2230                             if (os != 'Windows_NT') {
2231                                 return
2232                             }
2233                             break
2234                         case 'x64':
2235                             // Everything implemented
2236                             break
2237                         default:
2238                             println("Unknown architecture: ${architecture}")
2239                             assert false
2240                             break
2241                     }
2242
2243                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2244                     // with checked builds)
2245                     if (isJitStressScenario(scenario)) {
2246                         if (configuration != 'Checked') {
2247                             return
2248                         }
2249
2250                         // Since these are just execution time differences,
2251                         // skip platforms that don't execute the tests here (Windows_NT only)
2252                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2253                         if (!isEnabledOS || isBuildOnly) {
2254                             return
2255                         }
2256
2257                         switch (architecture) {
2258                             case 'x64':
2259                             case 'x86':
2260                             case 'x86_arm_altjit':
2261                             case 'x64_arm64_altjit':
2262                                 // x86 ubuntu: default only
2263                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2264                                     return
2265                                 }
2266                                 // Windows: Everything implemented
2267                                 break
2268
2269                             default:
2270                                 // arm, arm64, armlb: stress is handled through flow jobs.
2271                                 return
2272                         }
2273                     }
2274                     else if (isR2RScenario(scenario)) {
2275                         if (os != 'Windows_NT') {
2276                             return
2277                         }
2278                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2279                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2280                             return
2281                         }
2282                     }
2283                     else {
2284                         // Skip scenarios
2285                         switch (scenario) {
2286                             case 'ilrt':
2287                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2288                                 // the default scenario build
2289                                 if (os != 'Windows_NT') {
2290                                     return
2291                                 }
2292                                 // Only x64 for now
2293                                 if (architecture != 'x64') {
2294                                     return
2295                                 }
2296                                 // Release only
2297                                 if (configuration != 'Release') {
2298                                     return
2299                                 }
2300                                 break
2301                             case 'jitdiff':
2302                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2303                                     return
2304                                 }
2305                                 if (architecture != 'x64') {
2306                                     return
2307                                 }
2308                                 if (configuration != 'Checked') {
2309                                     return
2310                                 }
2311                                 break
2312                             case 'longgc':
2313                             case 'gcsimulator':
2314                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2315                                     return
2316                                 }
2317                                 if (architecture != 'x64') {
2318                                     return
2319                                 }
2320                                 if (configuration != 'Release') {
2321                                     return
2322                                 }
2323                                 break
2324                             case 'gc_reliability_framework':
2325                             case 'standalone_gc':
2326                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2327                                     return
2328                                 }
2329
2330                                 if (architecture != 'x64') {
2331                                     return
2332                                 }
2333
2334                                 if (configuration != 'Release' && configuration != 'Checked') {
2335                                     return
2336                                 }
2337                                 break
2338                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2339                             case 'formatting':
2340                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2341                                     return
2342                                 }
2343                                 if (architecture != 'x64') {
2344                                     return
2345                                 }
2346                                 if (configuration != 'Checked') {
2347                                     return
2348                                 }
2349                                 if (isBuildOnly) {
2350                                     return
2351                                 }
2352                                 break
2353                             case 'illink':
2354                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2355                                     return
2356                                 }
2357                                 if (architecture != 'x64' && architecture != 'x86') {
2358                                     return
2359                                 }
2360                                 if (isBuildOnly) {
2361                                     return
2362                                 }
2363                                 break
2364                             case 'normal':
2365                                 // Nothing skipped
2366                                 break
2367                             case 'innerloop':
2368                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2369                                     return
2370                                 }
2371                                 break
2372                             default:
2373                                 println("Unknown scenario: ${scenario}")
2374                                 assert false
2375                                 break
2376                         }
2377                     }
2378
2379                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2380                     // runtime behavior, not compile-time behavior, are not interesting.
2381                     switch (architecture) {
2382                         case 'x86_arm_altjit':
2383                         case 'x64_arm64_altjit':
2384                             if (isGCStressRelatedTesting(scenario)) {
2385                                 return
2386                             }
2387                             break
2388                         default:
2389                             break
2390                     }
2391
2392                     // Calculate names
2393                     def lowerConfiguration = configuration.toLowerCase()
2394                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2395                     def folderName = getJobFolder(scenario)
2396
2397                     // Create the new job
2398                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2399                     addToViews(newJob, isPR, architecture, os)
2400
2401                     def machineAffinityOptions = null
2402
2403                     if (os != 'Windows_NT') {
2404                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2405                     }
2406                     else {
2407                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2408                     }
2409
2410                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2411
2412                     // Add all the standard options
2413                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2414                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2415
2416                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2417                     def osGroup = getOSGroup(os)
2418
2419                     newJob.with {
2420                         steps {
2421                             if (os == 'Windows_NT') {
2422                                 buildCommands.each { buildCommand ->
2423                                     batchFile(buildCommand)
2424                                 }
2425                             }
2426                             else {
2427                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2428                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2429                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2430                                     // Define the Windows Tests and Corefx build job names
2431                                     def WindowsTestsName = projectFolder + '/' +
2432                                                            Utilities.getFullJobName(project,
2433                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2434                                                                                     false)
2435                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2436                                                        Utilities.getFolderName(branch)
2437
2438                                     // Copy the Windows test binaries and the Corefx build binaries
2439                                     copyArtifacts(WindowsTestsName) {
2440                                         includePatterns('bin/tests/tests.zip')
2441                                         buildSelector {
2442                                             latestSuccessful(true)
2443                                         }
2444                                     }
2445
2446                                     def arm_abi = 'arm'
2447                                     def corefx_os = 'linux'
2448                                     if (os == 'Tizen') {
2449                                         arm_abi = 'armel'
2450                                         corefx_os = 'tizen'
2451                                     }
2452
2453                                     // Let's use release CoreFX to test checked CoreCLR,
2454                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2455                                     def corefx_lowerConfiguration = lowerConfiguration
2456                                     if ( lowerConfiguration == 'checked' ) {
2457                                         corefx_lowerConfiguration='release'
2458                                     }
2459
2460                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2461                                         includePatterns('bin/build.tar.gz')
2462                                         buildSelector {
2463                                             latestSuccessful(true)
2464                                         }
2465                                     }
2466                                 }
2467
2468                                 buildCommands.each { buildCommand ->
2469                                     shell(buildCommand)
2470                                 }
2471                             }
2472                         }
2473                     } // newJob.with
2474
2475                 } // os
2476             } // configuration
2477         } // architecture
2478     } // isPR
2479 } // scenario
2480
2481
2482 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2483 Constants.allScenarios.each { scenario ->
2484     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2485
2486     [true, false].each { isPR ->
2487         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2488             Constants.crossList.each { os ->
2489                 if (architecture == 'arm64') {
2490                     if (os != "Ubuntu" && os != "Windows_NT") {
2491                         return
2492                     }
2493                 } else if (architecture == 'arm' || architecture == 'armlb') {
2494                     if (os != 'Windows_NT') {
2495                         return
2496                     }
2497                 } else if (architecture == 'x86') {
2498                     if (os != "Ubuntu") {
2499                         return
2500                     }
2501                 }
2502
2503                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2504
2505                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2506                     return
2507                 }
2508
2509                 Constants.configurationList.each { configuration ->
2510
2511                     // First, filter based on OS.
2512
2513                     if (os == 'Windows_NT') {
2514                         if (!isArmWindowsScenario(scenario)) {
2515                             return
2516                         }
2517                     }
2518                     else {
2519                         // Non-Windows
2520                         if (architecture == 'arm64') {
2521                             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
2522                                 return
2523                             }
2524                         }
2525                         else if (architecture == 'x86') {
2526                             // Linux/x86 only want innerloop and default test
2527                             if (!isNormalOrInnerloop) {
2528                                 return
2529                             }
2530                         }
2531                     }
2532
2533                     // For CentOS, we only want Checked/Release builds.
2534                     if (os == 'CentOS7.1') {
2535                         if (configuration != 'Checked' && configuration != 'Release') {
2536                             return
2537                         }
2538                         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2539                             return
2540                         }
2541                     }
2542
2543                     // For RedHat and Debian, we only do Release builds.
2544                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2545                         if (configuration != 'Release') {
2546                             return
2547                         }
2548                         if (!isNormalOrInnerloop) {
2549                             return
2550                         }
2551                     }
2552
2553                     // Next, filter based on scenario.
2554
2555                     if (isJitStressScenario(scenario)) {
2556                         if (configuration != 'Checked') {
2557                             return
2558                         }
2559                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2560                         if (isCoreFxScenario(scenario)) {
2561                             return
2562                         }
2563                     }
2564                     else if (isR2RBaselineScenario(scenario)) {
2565                         if (configuration != 'Checked' && configuration != 'Release') {
2566                             return
2567                         }
2568                     }
2569                     else if (isR2RStressScenario(scenario)) {
2570                         if (configuration != 'Checked') {
2571                             return
2572                         }
2573                     }
2574                     else {
2575                         // Skip scenarios
2576                         switch (scenario) {
2577                             case 'ilrt':
2578                             case 'longgc':
2579                             case 'gcsimulator':
2580                                 // Long GC tests take a long time on non-Release builds
2581                                 // ilrt is also Release only
2582                                 if (configuration != 'Release') {
2583                                     return
2584                                 }
2585                                 break
2586
2587                             case 'jitdiff':
2588                                 if (configuration != 'Checked') {
2589                                     return;
2590                                 }
2591                                 break
2592
2593                             case 'gc_reliability_framework':
2594                             case 'standalone_gc':
2595                                 if (configuration != 'Release' && configuration != 'Checked') {
2596                                     return
2597                                 }
2598                                 break
2599
2600                             case 'formatting':
2601                                 return
2602                             case 'illink':
2603                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2604                                     return
2605                                 }
2606                                 break
2607
2608                             case 'normal':
2609                                 // Nothing skipped
2610                                 break
2611
2612                             case 'innerloop':
2613                                 // Nothing skipped
2614                                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
2615                                     return
2616                                 }
2617                                 break
2618
2619                             default:
2620                                 println("Unknown scenario: ${scenario}")
2621                                 assert false
2622                                 break
2623                         }
2624                     }
2625
2626                     // Done filtering. Now, create the jobs.
2627
2628                     def lowerConfiguration = configuration.toLowerCase()
2629                     def osGroup = getOSGroup(os)
2630                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2631
2632                     def buildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2633
2634                     def inputCoreCLRBuildName = projectFolder + '/' +
2635                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, buildScenario, false), isPR)
2636
2637                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2638                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2639                     def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
2640                     def inputWindowsTestBuildArch = architecture
2641                     if (architecture == "arm64" && os != "Windows_NT") {
2642                         // Use the x64 test build for arm64 unix
2643                         inputWindowsTestBuildArch = "x64"
2644                     }
2645
2646                     def inputWindowsTestsBuildName = ""
2647
2648                     if (isJitStressScenario(scenario)) {
2649                         inputWindowsTestsBuildName = projectFolder + '/' +
2650                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2651                     } else {
2652                         inputWindowsTestsBuildName = projectFolder + '/' +
2653                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2654
2655                     }
2656
2657
2658                     // Enable Server GC for Ubuntu PR builds
2659                     def serverGCString = ''
2660                     if (os == 'Ubuntu' && isPR) {
2661                         serverGCString = '--useServerGC'
2662                     }
2663
2664                     def testOpts = ''
2665
2666                     if (isR2RScenario(scenario)) {
2667
2668                         testOpts += ' --crossgen --runcrossgentests'
2669
2670                         if (scenario == 'r2r_jitstress1') {
2671                             testOpts += ' --jitstress=1'
2672                         }
2673                         else if (scenario == 'r2r_jitstress2') {
2674                             testOpts += ' --jitstress=2'
2675                         }
2676                         else if (scenario == 'r2r_jitstressregs1') {
2677                             testOpts += ' --jitstressregs=1'
2678                         }
2679                         else if (scenario == 'r2r_jitstressregs2') {
2680                             testOpts += ' --jitstressregs=2'
2681                         }
2682                         else if (scenario == 'r2r_jitstressregs3') {
2683                             testOpts += ' --jitstressregs=3'
2684                         }
2685                         else if (scenario == 'r2r_jitstressregs4') {
2686                             testOpts += ' --jitstressregs=4'
2687                         }
2688                         else if (scenario == 'r2r_jitstressregs8') {
2689                             testOpts += ' --jitstressregs=8'
2690                         }
2691                         else if (scenario == 'r2r_jitstressregs0x10') {
2692                             testOpts += ' --jitstressregs=0x10'
2693                         }
2694                         else if (scenario == 'r2r_jitstressregs0x80') {
2695                             testOpts += ' --jitstressregs=0x80'
2696                         }
2697                         else if (scenario == 'r2r_jitstressregs0x1000') {
2698                             testOpts += ' --jitstressregs=0x1000'
2699                         }
2700                         else if (scenario == 'r2r_jitminopts') {
2701                             testOpts += ' --jitminopts'
2702                         }
2703                         else if (scenario == 'r2r_jitforcerelocs') {
2704                             testOpts += ' --jitforcerelocs'
2705                         }
2706                         else if (scenario == 'r2r_gcstress15') {
2707                             testOpts += ' --gcstresslevel=0xF'
2708                         }
2709                     }
2710                     else if (scenario == 'jitdiff') {
2711                         testOpts += ' --jitdisasm --crossgen'
2712                     }
2713                     else if (scenario == 'illink') {
2714                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2715                     }
2716                     else if (isLongGc(scenario)) {
2717                         // Long GC tests behave very poorly when they are not
2718                         // the only test running (many of them allocate until OOM).
2719                         testOpts += ' --sequential'
2720
2721                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2722                         // for running long GC and GCSimulator tests, respectively. We don't use them
2723                         // here because using a playlist file produces much more readable output on the CI machines
2724                         // and reduces running time.
2725                         //
2726                         // The Long GC playlist contains all of the tests that are
2727                         // going to be run. The GCSimulator playlist contains all of
2728                         // the GC simulator tests.
2729                         if (scenario == 'longgc') {
2730                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2731                         }
2732                         else if (scenario == 'gcsimulator') {
2733                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2734                         }
2735                     }
2736                     else if (isGcReliabilityFramework(scenario)) {
2737                         testOpts += ' --build-overlay-only'
2738                     }
2739                     else if (scenario == 'standalone_gc') {
2740                         if (osGroup == 'OSX') {
2741                             testOpts += ' --gcname=libclrgc.dylib'
2742                         }
2743                         else if (osGroup == 'Linux') {
2744                             testOpts += ' --gcname=libclrgc.so'
2745                         }
2746                         else {
2747                             println("Unexpected OS group: ${osGroup} for os ${os}")
2748                             assert false
2749                         }
2750                     }
2751
2752                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2753
2754                     def folder = getJobFolder(scenario)
2755                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2756                         // Add parameters for the inputs
2757
2758                         if (windowsArmJob == true) {
2759                             parameters {
2760                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2761                             }
2762                         }
2763                         else {
2764                             parameters {
2765                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2766                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2767                             }
2768                         }
2769
2770                         steps {
2771                             // Set up the copies
2772
2773                             // Coreclr build containing the tests and mscorlib
2774                             // pri1 jobs still need to copy windows_nt built tests
2775                             if (windowsArmJob != true) {
2776                                 copyArtifacts(inputWindowsTestsBuildName) {
2777                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2778                                     buildSelector {
2779                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2780                                     }
2781                                 }
2782                             }
2783
2784                             // Coreclr build we are trying to test
2785                             //
2786                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2787
2788                             copyArtifacts(inputCoreCLRBuildName) {
2789                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2790                                 buildSelector {
2791                                     buildNumber('${CORECLR_BUILD}')
2792                                 }
2793                             }
2794
2795                             // Windows CoreCLR Arm(64) will restore corefx
2796                             // packages correctly.
2797                             //
2798                             // In addition, test steps are entirely different
2799                             // because we do not have a unified runner
2800                             if (windowsArmJob != true) {
2801                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2802
2803                                 // HACK -- Arm64 does not have corefx jobs yet.
2804                                 // Clone corefx and build the native packages overwriting the x64 packages.
2805                                 if (architecture == 'arm64') {
2806                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2807                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2808                                 }
2809                                 else if (architecture == 'x86') {
2810                                     shell("mkdir ./bin/CoreFxNative")
2811
2812                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2813                                         includePatterns('bin/build.tar.gz')
2814                                         targetDirectory('bin/CoreFxNative')
2815                                         buildSelector {
2816                                             latestSuccessful(true)
2817                                         }
2818                                     }
2819
2820                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2821                                 }
2822
2823                                 // Unzip the tests first.  Exit with 0
2824                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2825                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2826
2827                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2828
2829                                 // Execute the tests
2830                                 def runDocker = isNeedDocker(architecture, os, false)
2831                                 def dockerPrefix = ""
2832                                 def dockerCmd = ""
2833                                 if (runDocker) {
2834                                     def dockerImage = getDockerImageName(architecture, os, false)
2835                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2836                                     dockerCmd = dockerPrefix + "${dockerImage} "
2837                                 }
2838
2839                                 // If we are running a stress mode, we'll set those variables first
2840                                 def testEnvOpt = ""
2841                                 if (isJitStressScenario(scenario)) {
2842                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2843                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2844                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2845                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2846                                     shell("${envScriptCmds}")
2847                                     testEnvOpt = "--test-env=" + scriptFileName
2848                                 }
2849
2850                                 if (isGCStressRelatedTesting(scenario)) {
2851                                     shell('./init-tools.sh')
2852                                 }
2853
2854                                 shell("""${dockerCmd}./tests/runtest.sh \\
2855                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2856                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2857                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2858                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2859
2860                                 if (isGcReliabilityFramework(scenario)) {
2861                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2862                                     if (serverGCString != '') {
2863                                         if (runDocker) {
2864                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2865                                         }
2866                                         else {
2867                                             shell("export COMPlus_gcServer=1")
2868                                         }
2869                                     }
2870
2871                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2872                                 }
2873                             }
2874
2875                             else { // windowsArmJob == true
2876                                 // Unzip tests.
2877                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2878
2879                                 // Build the build commands
2880                                 def buildCommands = ""
2881
2882                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2883                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2884                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2885
2886                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2887                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2888
2889                                 // For all jobs
2890                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2891
2892                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2893                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2894
2895                                 // ARM legacy backend; this is an altjit.
2896                                 if (architecture == "armlb") {
2897                                     addEnvVariable("COMPlus_AltJit", "*")
2898                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2899                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2900                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2901                                 }
2902
2903                                 // If we are running a stress mode, we'll set those variables as well
2904                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2905                                     def stressValues = null
2906                                     if (isJitStressScenario(scenario)) {
2907                                         stressValues = Constants.jitStressModeScenarios[scenario]
2908                                     }
2909                                     else {
2910                                         stressValues = Constants.r2rStressScenarios[scenario]
2911                                     }
2912
2913                                     stressValues.each { key, value ->
2914                                         addEnvVariable(key, value)
2915                                     }
2916                                 }
2917
2918                                 if (isR2RScenario(scenario)) {
2919                                     // Crossgen the framework assemblies.
2920                                     buildCommands += """
2921 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2922 @goto skip_PrecompileAssembly
2923
2924 :PrecompileAssembly
2925 @REM Skip mscorlib since it is already precompiled.
2926 @if /I "%3" == "mscorlib.dll" exit /b 0
2927 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2928
2929 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2930 @if "%errorlevel%" == "-2146230517" (
2931     echo %2 is not a managed assembly.
2932 ) else if "%errorlevel%" == "-2146234344" (
2933     echo %2 is not a managed assembly.
2934 ) else if %errorlevel% neq 0 (
2935     echo Unable to precompile %2
2936 ) else (
2937     echo Precompiled %2
2938 )
2939 @exit /b 0
2940
2941 :skip_PrecompileAssembly
2942 """
2943
2944                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2945                                     // crossgen on tests before running them.
2946                                     addEnvVariable("RunCrossGen", "true")
2947                                 }
2948
2949                                 // Create the smarty command
2950                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2951                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2952                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2953
2954                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2955
2956                                 if (architecture == "armlb") {
2957                                     addExclude("LEGACYJIT_FAIL")
2958                                 }
2959
2960                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2961                                     def failTag = "JITSTRESS_FAIL"
2962                                     def excludeTag = "JITSTRESS_EXCLUDE"
2963
2964                                     if (scenario.contains('gc')) {
2965                                         failTag = "GCSTRESS_FAIL"
2966                                         excludeTag = "GCSTRESS_EXCLUDE"
2967                                     }
2968
2969                                     addArchSpecificExclude(architecture, failTag)
2970                                     addArchSpecificExclude(architecture, excludeTag)
2971                                 }
2972                                 else {
2973                                     addExclude("pri1")
2974                                 }
2975
2976                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2977                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2978                                 addExclude("LONG_RUNNING")
2979
2980                                 smartyCommand += "/lstFile Tests.lst"
2981
2982                                 def testListArch = [
2983                                     'arm64': 'arm64',
2984                                     'arm': 'arm',
2985                                     'armlb': 'arm'
2986                                 ]
2987
2988                                 def archLocation = testListArch[architecture]
2989
2990                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2991                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2992                                 addCommand("${smartyCommand}")
2993
2994                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2995                                 // However, we also need to remove all the variables that were set during this batch file, so we
2996                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2997                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2998                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2999                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3000                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3001                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3002
3003                                 // ZIP up the smarty output, no matter what the smarty result.
3004                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3005
3006                                 addCommand("echo %errorlevel%")
3007                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3008
3009                                 // Use the smarty errorlevel as the script errorlevel.
3010                                 addCommand("exit /b %__save_smarty_errorlevel%")
3011
3012                                 batchFile(buildCommands)
3013                             }
3014                         }
3015                     }
3016
3017                     addToViews(newJob, isPR, architecture, os)
3018
3019                     if (scenario == 'jitdiff') {
3020                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3021                     }
3022
3023                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3024                     if (os in ['Ubuntu']) {
3025                         SummaryBuilder summaries = new SummaryBuilder()
3026                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3027                         summaries.emit(newJob)
3028                     }
3029
3030                     def affinityOptions = null
3031
3032                     if (windowsArmJob == true) {
3033                         affinityOptions = [
3034                             "use_arm64_build_machine" : false
3035                         ]
3036                     }
3037
3038                     else if (architecture == 'arm64' && os != 'Windows_NT') {
3039                         affinityOptions = [
3040                             "large_pages" : false
3041                         ]
3042                     }
3043
3044                     setMachineAffinity(newJob, os, architecture, affinityOptions)
3045                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3046
3047                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3048
3049                     if (windowsArmJob != true) {
3050                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3051                     }
3052                     else {
3053                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3054
3055                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3056                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3057                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3058                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3059                     }
3060
3061                     // Create a build flow to join together the build and tests required to run this test.
3062                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3063                     // Linux CoreCLR test
3064                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3065                     def fullTestJobName = projectFolder + '/' + newJob.name
3066                     // Add a reference to the input jobs for report purposes
3067                     JobReport.Report.addReference(inputCoreCLRBuildName)
3068                     JobReport.Report.addReference(inputWindowsTestsBuildName)
3069                     JobReport.Report.addReference(fullTestJobName)
3070                     def newFlowJob = null
3071
3072                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3073                         // Do not create the flow job for RHEL jobs.
3074                         return
3075                     }
3076
3077                     // For pri0 jobs we can build tests on unix
3078                     if (windowsArmJob) {
3079                         // For Windows arm jobs there is no reason to build a parallel test job.
3080                         // The product build supports building and archiving the tests.
3081
3082                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3083                         buildFlow("""
3084 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3085
3086 // And then build the test build
3087 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3088 """)
3089                         }
3090                     }
3091                     else {
3092                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
3093                         buildFlow("""
3094 // Build the input jobs in parallel
3095 parallel (
3096 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3097 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
3098 )
3099
3100 // And then build the test build
3101 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3102             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3103 """)
3104                         }
3105                     }
3106
3107                     addToViews(newFlowJob, isPR, architecture, os)
3108
3109                     // For the flow jobs set the machine affinity as x64 if an armarch.
3110                     def flowArch = architecture
3111
3112                     if (flowArch in validWindowsNTCrossArches) {
3113                         flowArch = 'x64'
3114                         affinityOptions = null
3115                     }
3116
3117                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
3118                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3119                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3120                 } // configuration
3121             } // os
3122         } // architecture
3123     } // isPR
3124 } // scenario
3125
3126 JobReport.Report.generateJobReport(out)
3127
3128 // Make the call to generate the help job
3129 Utilities.createHelperJob(this, project, branch,
3130     "Welcome to the ${project} Repository",  // This is prepended to the help message
3131     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3132
3133 Utilities.addCROSSCheck(this, project, branch)