netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  39     def static LimitedHardwareBranches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  73                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  74                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  75                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  76                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  77                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  78                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  79                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  80                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  81                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  82                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  83                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  84                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  85                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  86                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
  87                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
  88                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  89                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  90                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  91                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  92                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  93                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  94                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  95                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  96                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
  97                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  98                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
  99                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 100                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 101                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 102                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 103                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 104                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 105                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 106                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 107                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 108                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 109                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 110                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 111                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 112                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 113                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 114                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 115                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 116                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 117                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 118                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 119                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 120                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 121                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 122                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 123                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 124                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 125                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 126                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 127     ]
 128
 129     // This is a set of ReadyToRun stress scenarios
 130     def static r2rStressScenarios = [
 131                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 132                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 133                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 134                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 135                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 136                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 137                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 138                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 139                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 140                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 141                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 142                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 143                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 144                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 145                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 146                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 147     ]
 148
 149     // This is the basic set of scenarios
 150     def static basicScenarios = [
 151                'innerloop',
 152                'normal',
 153                'ilrt',
 154                'r2r',
 155                'longgc',
 156                'formatting',
 157                'gcsimulator',
 158                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 159                'standalone_gc',
 160                'gc_reliability_framework',
 161                'illink',
 162                'corefx_innerloop']
 163
 164     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 165
 166     // Valid PR trigger combinations.
 167     def static prTriggeredValidInnerLoopCombos = [
 168         'Windows_NT': [
 169             'x64': [
 170                 'Checked'
 171             ],
 172             'x86': [
 173                 'Checked',
 174                 'Release'
 175             ],
 176             'arm': [
 177                 'Checked',
 178             ],
 179             'arm64': [
 180                 'Checked'
 181             ]
 182         ],
 183         'Windows_NT_BuildOnly': [
 184             'x64': [
 185                 'Checked',
 186                 'Release'
 187             ],
 188             'x86': [
 189                 'Checked',
 190                 'Release'
 191             ],
 192             'arm': [
 193                 'Checked'
 194             ],
 195         ],
 196         'Ubuntu': [
 197             'x64': [
 198                 'Checked'
 199             ],
 200             'arm64': [
 201                 'Debug'
 202             ],
 203             'arm': [
 204                 'Checked'
 205             ]
 206         ],
 207         'CentOS7.1': [
 208             'x64': [
 209                 'Debug',
 210                 'Checked'
 211             ]
 212         ],
 213         'OSX10.12': [
 214             'x64': [
 215                 'Checked'
 216             ]
 217         ],
 218         'Tizen': [
 219             'armem': [
 220                 'Checked'
 221             ]
 222         ]
 223     ]
 224
 225     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 226     // to Tests.lst file categories to exclude.
 227     //
 228     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 229     // and with the same values, with some commented out, for easier maintenance.
 230     //
 231     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 232     //
 233     def static validArmWindowsScenarios = [
 234                'innerloop':                              [],
 235                'normal':                                 [],
 236                // 'ilrt'
 237                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 238                // 'longgc'
 239                // 'formatting'
 240                // 'gcsimulator'
 241                // 'jitdiff'
 242                // 'standalone_gc'
 243                // 'gc_reliability_framework'
 244                // 'illink'
 245                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 258                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 260                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 261                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 262                'tieredcompilation':                      [],
 263                'no_tiered_compilation':                  [],
 264                'no_tiered_compilation_innerloop':        [],
 265                'forcerelocs':                            [],
 266                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 277                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 278                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 279                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 280                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 281                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 282                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 283                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 284                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 285                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 287                // 'jitsse2only'                          // Only relevant to xarch
 288                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 289                // 'jitincompletehwintrinsic'
 290                // 'jitx86hwintrinsicnoavx'
 291                // 'jitx86hwintrinsicnoavx2'
 292                // 'jitx86hwintrinsicnosimd'
 293                // 'jitnox86hwintrinsic'
 294                'corefx_baseline':                        [],    // corefx tests don't use smarty
 295                'corefx_minopts':                         [],    // corefx tests don't use smarty
 296                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 297                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 298                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 299                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 300                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 301                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 302                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 303                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 304                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 305                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 306                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 307                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 308                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 309                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 310                'heapverify1':                            [],
 311                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 312                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 314                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 317
 318                //
 319                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 320                //
 321
 322                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 323                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 324                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 325                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 326                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 327                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 328                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 329                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 330                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 331                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 332                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 333                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 334                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 336                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 337                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 338                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 339                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 340                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 341     ]
 342
 343     def static validLinuxArmScenarios = [
 344                'innerloop',
 345                'normal',
 346                // 'ilrt'
 347                'r2r',
 348                // 'longgc'
 349                // 'formatting'
 350                // 'gcsimulator'
 351                // 'jitdiff'
 352                // 'standalone_gc'
 353                // 'gc_reliability_framework'
 354                // 'illink'
 355                'r2r_jitstress1',
 356                'r2r_jitstress2',
 357                'r2r_jitstress1_tiered',
 358                'r2r_jitstress2_tiered',
 359                'r2r_jitstressregs1',
 360                'r2r_jitstressregs2',
 361                'r2r_jitstressregs3',
 362                'r2r_jitstressregs4',
 363                'r2r_jitstressregs8',
 364                'r2r_jitstressregs0x10',
 365                'r2r_jitstressregs0x80',
 366                'r2r_jitstressregs0x1000',
 367                'r2r_jitminopts',
 368                'r2r_jitforcerelocs',
 369                'r2r_gcstress15',
 370                'r2r_no_tiered_compilation',
 371                'minopts',
 372                'tieredcompilation',
 373                'no_tiered_compilation',
 374                'no_tiered_compilation_innerloop',
 375                'forcerelocs',
 376                'jitstress1',
 377                'jitstress2',
 378                'jitstress1_tiered',
 379                'jitstress2_tiered',
 380                'jitstressregs1',
 381                'jitstressregs2',
 382                'jitstressregs3',
 383                'jitstressregs4',
 384                'jitstressregs8',
 385                'jitstressregs0x10',
 386                'jitstressregs0x80',
 387                'jitstressregs0x1000',
 388                'jitstress2_jitstressregs1',
 389                'jitstress2_jitstressregs2',
 390                'jitstress2_jitstressregs3',
 391                'jitstress2_jitstressregs4',
 392                'jitstress2_jitstressregs8',
 393                'jitstress2_jitstressregs0x10',
 394                'jitstress2_jitstressregs0x80',
 395                'jitstress2_jitstressregs0x1000',
 396                'tailcallstress',
 397                // 'jitsse2only'                          // Only relevant to xarch
 398                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 399                // 'jitincompletehwintrinsic'
 400                // 'jitx86hwintrinsicnoavx'
 401                // 'jitx86hwintrinsicnoavx2'
 402                // 'jitx86hwintrinsicnosimd'
 403                // 'jitnox86hwintrinsic'
 404                'corefx_baseline',
 405                'corefx_minopts',
 406                'corefx_tieredcompilation',
 407                'corefx_jitstress1',
 408                'corefx_jitstress2',
 409                'corefx_jitstressregs1',
 410                'corefx_jitstressregs2',
 411                'corefx_jitstressregs3',
 412                'corefx_jitstressregs4',
 413                'corefx_jitstressregs8',
 414                'corefx_jitstressregs0x10',
 415                'corefx_jitstressregs0x80',
 416                'corefx_jitstressregs0x1000',
 417                'gcstress0x3',
 418                'gcstress0xc',
 419                'zapdisable',
 420                'heapverify1',
 421                'gcstress0xc_zapdisable',
 422                'gcstress0xc_zapdisable_jitstress2',
 423                'gcstress0xc_zapdisable_heapverify1',
 424                'gcstress0xc_jitstress1',
 425                'gcstress0xc_jitstress2',
 426                'gcstress0xc_minopts_heapverify1'
 427     ]
 428
 429     def static validLinuxArm64Scenarios = [
 430                'innerloop',
 431                'normal',
 432                // 'ilrt'
 433                'r2r',
 434                // 'longgc'
 435                // 'formatting'
 436                // 'gcsimulator'
 437                // 'jitdiff'
 438                // 'standalone_gc'
 439                // 'gc_reliability_framework'
 440                // 'illink'
 441                'r2r_jitstress1',
 442                'r2r_jitstress2',
 443                'r2r_jitstress1_tiered',
 444                'r2r_jitstress2_tiered',
 445                'r2r_jitstressregs1',
 446                'r2r_jitstressregs2',
 447                'r2r_jitstressregs3',
 448                'r2r_jitstressregs4',
 449                'r2r_jitstressregs8',
 450                'r2r_jitstressregs0x10',
 451                'r2r_jitstressregs0x80',
 452                'r2r_jitstressregs0x1000',
 453                'r2r_jitminopts',
 454                'r2r_jitforcerelocs',
 455                'r2r_gcstress15',
 456                'minopts',
 457                'tieredcompilation',
 458                'forcerelocs',
 459                'jitstress1',
 460                'jitstress2',
 461                'jitstress1_tiered',
 462                'jitstress2_tiered',
 463                'jitstressregs1',
 464                'jitstressregs2',
 465                'jitstressregs3',
 466                'jitstressregs4',
 467                'jitstressregs8',
 468                'jitstressregs0x10',
 469                'jitstressregs0x80',
 470                'jitstressregs0x1000',
 471                'jitstress2_jitstressregs1',
 472                'jitstress2_jitstressregs2',
 473                'jitstress2_jitstressregs3',
 474                'jitstress2_jitstressregs4',
 475                'jitstress2_jitstressregs8',
 476                'jitstress2_jitstressregs0x10',
 477                'jitstress2_jitstressregs0x80',
 478                'jitstress2_jitstressregs0x1000',
 479                'tailcallstress',
 480                // 'jitsse2only'                         // Only relevant to xarch
 481                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 482                // 'jitincompletehwintrinsic'
 483                // 'jitx86hwintrinsicnoavx'
 484                // 'jitx86hwintrinsicnoavx2'
 485                // 'jitx86hwintrinsicnosimd'
 486                // 'jitnox86hwintrinsic'
 487                'corefx_baseline',
 488                'corefx_minopts',
 489                'corefx_tieredcompilation',
 490                'corefx_jitstress1',
 491                'corefx_jitstress2',
 492                'corefx_jitstressregs1',
 493                'corefx_jitstressregs2',
 494                'corefx_jitstressregs3',
 495                'corefx_jitstressregs4',
 496                'corefx_jitstressregs8',
 497                'corefx_jitstressregs0x10',
 498                'corefx_jitstressregs0x80',
 499                'corefx_jitstressregs0x1000',
 500                'gcstress0x3',
 501                'gcstress0xc',
 502                'zapdisable',
 503                'heapverify1',
 504                'gcstress0xc_zapdisable',
 505                'gcstress0xc_zapdisable_jitstress2',
 506                'gcstress0xc_zapdisable_heapverify1',
 507                'gcstress0xc_jitstress1',
 508                'gcstress0xc_jitstress2',
 509                'gcstress0xc_minopts_heapverify1'
 510     ]
 511
 512     def static configurationList = ['Debug', 'Checked', 'Release']
 513
 514     // This is the set of architectures
 515     // Some of these are pseudo-architectures:
 516     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 517     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 518     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 519     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 520
 521     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 522     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 523 }
 524
 525 // **************************************************************
 526 // Create some specific views
 527 //
 528 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 529 // views based on a single regular expression. These views will be generated by adding a
 530 // specific set of jobs to them.
 531 //
 532 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 533 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 534 // those are required, don't add them (which simplifies the view pages, as well).
 535 // **************************************************************
 536
 537 class Views {
 538     def static MergeJobView = null
 539     def static PeriodicJobView = null
 540     def static ArchitectureViews = [:]
 541     def static OSViews = [:]
 542 }
 543
 544 // MergeJobView: include all jobs that execute when a PR change is merged.
 545 Views.MergeJobView = listView('Merge') {
 546     recurse()
 547     columns {
 548         status()
 549         weather()
 550         name()
 551         lastSuccess()
 552         lastFailure()
 553         lastDuration()
 554         buildButton()
 555     }
 556 }
 557
 558 // PeriodicJobView: include all jobs that execute on a schedule
 559 Views.PeriodicJobView = listView('Periodic') {
 560     recurse()
 561     columns {
 562         status()
 563         weather()
 564         name()
 565         lastSuccess()
 566         lastFailure()
 567         lastDuration()
 568         buildButton()
 569     }
 570 }
 571
 572 // Create a view for non-PR jobs for each architecture.
 573 Constants.architectureList.each { architecture ->
 574     Views.ArchitectureViews[architecture] = listView(architecture) {
 575         recurse()
 576         columns {
 577             status()
 578             weather()
 579             name()
 580             lastSuccess()
 581             lastFailure()
 582             lastDuration()
 583             buildButton()
 584         }
 585     }
 586 }
 587
 588 // Create a view for non-PR jobs for each OS.
 589 Constants.osList.each { os ->
 590     // Don't create one for the special 'Windows_NT_BuildOnly'
 591     if (os == 'Windows_NT_BuildOnly') {
 592         return
 593     }
 594     Views.OSViews[os] = listView(os) {
 595         recurse()
 596         columns {
 597             status()
 598             weather()
 599             name()
 600             lastSuccess()
 601             lastFailure()
 602             lastDuration()
 603             buildButton()
 604         }
 605     }
 606 }
 607
 608 def static addToMergeView(def job) {
 609     Views.MergeJobView.with {
 610         jobs {
 611             name(job.name)
 612         }
 613     }
 614 }
 615
 616 def static addToPeriodicView(def job) {
 617     Views.PeriodicJobView.with {
 618         jobs {
 619             name(job.name)
 620         }
 621     }
 622 }
 623
 624 def static addToViews(def job, def isPR, def architecture, def os) {
 625     if (isPR) {
 626         // No views want PR jobs currently.
 627         return
 628     }
 629
 630     // Add to architecture view.
 631     Views.ArchitectureViews[architecture].with {
 632         jobs {
 633             name(job.name)
 634         }
 635     }
 636
 637     // Add to OS view.
 638     Views.OSViews[os].with {
 639         jobs {
 640             name(job.name)
 641         }
 642     }
 643 }
 644
 645 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 646     addToPeriodicView(job)
 647     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 648 }
 649
 650 def static addGithubPushTriggerHelper(def job) {
 651     addToMergeView(job)
 652     Utilities.addGithubPushTrigger(job)
 653 }
 654
 655
 656 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 657     assert os instanceof String
 658     assert architecture instanceof String
 659
 660     def armArches = ['arm', 'armem', 'arm64']
 661
 662     if (!(architecture in armArches)) {
 663         assert options == null
 664         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 665
 666         return
 667     }
 668
 669     // This is an arm(64) job.
 670     //
 671     // There are several options.
 672     //
 673     // Windows_NT
 674     //
 675     // Arm32 (Build) -> latest-arm64
 676     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 677     // Arm32 (Test)  -> arm64-windows_nt
 678     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 679     //
 680     // Arm64 (Build) -> latest-arm64
 681     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 682     // Arm64 (Test)  -> arm64-windows_nt
 683     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 684     //
 685     // Ubuntu
 686     //
 687     // Arm32 emulator (Build, Test) -> arm-cross-latest
 688     //       |-> os == "Tizen" && (architecture == "armem")
 689     //
 690     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 691     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 692     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 693     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 694     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 695     //       |-> os == "Ubuntu" && (architecture == "arm")
 696     //
 697     // Arm64 (Build) -> arm64-cross-latest
 698     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 699     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 700     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 701     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 702     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 703
 704     // This has to be a arm arch
 705     assert architecture in armArches
 706     if (os == "Windows_NT") {
 707         // arm32/arm64 Windows jobs share the same machines for now
 708         def isBuild = options['use_arm64_build_machine'] == true
 709
 710         if (isBuild == true) {
 711             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 712         } else {
 713             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 714         }
 715     } else {
 716         assert os != 'Windows_NT'
 717
 718         if (architecture == 'arm64') {
 719             assert os == 'Ubuntu'
 720             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 721             def isBuild = (options != null) && (options['is_build_job'] == true)
 722             if (isFlow || isBuild) {
 723                 // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 724                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 725                 // Arm64 hardware.
 726                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 727             } else {
 728                 // Arm64 Linux test machines
 729                 if ((options != null) && (options['large_pages'] == true)) {
 730                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 731                 } else {
 732                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 733                 }
 734             }
 735         }
 736         else if (architecture == 'armem') {
 737             // arm emulator (Tizen). Build and test on same machine,
 738             // using Docker.
 739             assert os == 'Tizen'
 740             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 741         }
 742         else {
 743             // arm Ubuntu on hardware.
 744             assert architecture == 'arm'
 745             assert os == 'Ubuntu'
 746             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 747             def isBuild = (options != null) && (options['is_build_job'] == true)
 748             if (isFlow || isBuild) {
 749                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 750                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 751                 // arm hardware.
 752                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 753             } else {
 754                 // arm Ubuntu test machine
 755                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 756                 // Utilities.setMachineAffinity. Just add the machine affinity
 757                 // manually. We specify the Helix queue name here.
 758                 job.with {
 759                     label('ubuntu.1404.arm32.open')
 760                 }
 761             }
 762         }
 763     }
 764 }
 765
 766 // setJobMachineAffinity: compute the machine affinity options for a job,
 767 // then set the job with those affinity options.
 768 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 769 {
 770     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 771            (!isBuildJob && isTestJob  && !isFlowJob) ||
 772            (!isBuildJob && !isTestJob && isFlowJob)
 773
 774     def affinityOptions = null
 775     def affinityArchitecture = architecture
 776
 777     if (os == "Windows_NT") {
 778         if (architecture in Constants.armWindowsCrossArchitectureList) {
 779             if (isBuildJob) {
 780                 affinityOptions = [ "use_arm64_build_machine" : true ]
 781             } else if (isTestJob) {
 782                 affinityOptions = [ "use_arm64_build_machine" : false ]
 783             } else if (isFlowJob) {
 784                 // For the flow jobs set the machine affinity as x64
 785                 affinityArchitecture = 'x64'
 786             }
 787         }
 788     }
 789     else {
 790         if (architecture == 'arm64') {
 791             if (isBuildJob) {
 792                 affinityOptions = ['is_build_job': true]
 793             } else if (isFlowJob) {
 794                 affinityOptions = ['is_flow_job': true]
 795             } else if (isTestJob) {
 796                 affinityOptions = [ "large_pages" : false ]
 797             }
 798         }
 799         else if (architecture == 'arm') {
 800             if (isBuildJob) {
 801                 affinityOptions = ['is_build_job': true]
 802             } else if (isFlowJob) {
 803                 affinityOptions = ['is_flow_job': true]
 804             }
 805         }
 806     }
 807
 808     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 809 }
 810
 811 def static isGCStressRelatedTesting(def scenario) {
 812     // The 'r2r_gcstress15' scenario is a basic scenario.
 813     // Detect it and make it a GCStress related.
 814     if (scenario == 'r2r_gcstress15')
 815     {
 816         return true;
 817     }
 818
 819     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 820     def scenarioName = scenario.toLowerCase()
 821     def isGCStressTesting = false
 822     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 823         if (k in gcStressTestEnvVars) {
 824             isGCStressTesting = true;
 825         }
 826     }
 827     return isGCStressTesting
 828 }
 829
 830 def static isCoreFxScenario(def scenario) {
 831     def corefx_prefix = 'corefx_'
 832     if (scenario.length() < corefx_prefix.length()) {
 833         return false
 834     }
 835     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 836 }
 837
 838 def static isR2RBaselineScenario(def scenario) {
 839     return (scenario == 'r2r')
 840 }
 841
 842 def static isR2RStressScenario(def scenario) {
 843     return Constants.r2rStressScenarios.containsKey(scenario)
 844 }
 845
 846 def static isR2RScenario(def scenario) {
 847     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 848 }
 849
 850 def static isJitStressScenario(def scenario) {
 851     return Constants.jitStressModeScenarios.containsKey(scenario)
 852 }
 853
 854 def static isLongGc(def scenario) {
 855     return (scenario == 'longgc' || scenario == 'gcsimulator')
 856 }
 857
 858 def static isJitDiff(def scenario) {
 859     return (scenario == 'jitdiff')
 860 }
 861
 862 def static isGcReliabilityFramework(def scenario) {
 863     return (scenario == 'gc_reliability_framework')
 864 }
 865
 866 def static isArmWindowsScenario(def scenario) {
 867     return Constants.validArmWindowsScenarios.containsKey(scenario)
 868 }
 869
 870 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 871     if (isBuildOnly == true) {
 872         os = 'Windows_NT_BuildOnly'
 873     }
 874
 875     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 876     if (validOsPrTriggerArchConfigs != null) {
 877         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 878         if (validOsPrTriggerConfigs != null) {
 879             if (configuration in validOsPrTriggerConfigs) {
 880                 return true
 881             }
 882         }
 883     }
 884
 885     return false
 886 }
 887
 888 // This means the job builds and runs the 'Pri0' test set. This does not mean the job is
 889 // scheduled with a default PR trigger despite the correlation being true at the moment.
 890 def static isPri0TestScenario(def scenario) {
 891     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 892 }
 893
 894 def static getFxBranch(def branch) {
 895     def fxBranch = branch
 896     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 897     // branch even though CoreFX doesn't have such a branch.
 898     if (branch == 'dev/unix_test_workflow') {
 899         fxBranch = 'master'
 900     }
 901     return fxBranch
 902 }
 903
 904 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 905     // 2 hours (120 minutes) is the default timeout
 906     def timeout = 120
 907
 908     if (!isPri0TestScenario(scenario)) {
 909         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 910         timeout = 240
 911     }
 912
 913     if (!isBuildOnly) {
 914         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 915         if (isGCStressRelatedTesting(scenario)) {
 916             timeout = 4320
 917         }
 918         else if (isCoreFxScenario(scenario)) {
 919             timeout = 360
 920         }
 921         else if (isJitStressScenario(scenario)) {
 922             timeout = 300
 923         }
 924         else if (isR2RBaselineScenario(scenario)) {
 925             timeout = 240
 926         }
 927         else if (isLongGc(scenario)) {
 928             timeout = 1440
 929         }
 930         else if (isJitDiff(scenario)) {
 931             timeout = 240
 932         }
 933         else if (isGcReliabilityFramework(scenario)) {
 934             timeout = 1440
 935         }
 936         else if (architecture == 'armem' || architecture == 'arm64') {
 937             timeout = 240
 938         }
 939
 940         if (architecture == 'arm') {
 941             // ARM32 machines are particularly slow.
 942             timeout += 120
 943         }
 944     }
 945
 946     if (configuration == 'Debug') {
 947         // Debug runs can be very slow. Add an hour.
 948         timeout += 60
 949     }
 950
 951     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 952         // AltJit runs compile all methods twice.
 953         timeout *= 2
 954     }
 955
 956     // If we've changed the timeout from the default, set it in the job.
 957
 958     if (timeout != 120) {
 959         Utilities.setJobTimeout(newJob, timeout)
 960     }
 961 }
 962
 963 def static getJobFolder(def scenario) {
 964     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 965         return 'jitstress'
 966     }
 967     if (scenario == 'illink') {
 968         return 'illink'
 969     }
 970     return ''
 971 }
 972
 973 def static getStressModeDisplayName(def scenario) {
 974     def displayStr = ''
 975     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 976         def prefixLength = 'COMPlus_'.length()
 977         if (k.length() >= prefixLength) {
 978             def modeName = k.substring(prefixLength, k.length())
 979             displayStr += ' ' + modeName + '=' + v
 980         }
 981     }
 982
 983     if (isCoreFxScenario(scenario)) {
 984         displayStr = ('CoreFx ' + displayStr).trim()
 985     }
 986
 987     return displayStr
 988 }
 989
 990 def static getR2RDisplayName(def scenario) {
 991     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 992     def displayStr = scenario
 993     def prefixLength = 'r2r_'.length()
 994     if (displayStr.length() >= prefixLength) {
 995         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 996     } else if (scenario == 'r2r') {
 997         displayStr = "R2R"
 998     }
 999     return displayStr
1000 }
1001
1002 //
1003 // Functions to create an environment script.
1004 //      envScriptCreate -- initialize the script (call first)
1005 //      envScriptFinalize -- finalize the script (call last)
1006 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1007 //      envScriptAppendExistingScript -- append an existing script to the generated script
1008 //
1009 // Each script returns a string of commands. Concatenate all the strings together before
1010 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1011 //
1012
1013 // Initialize the environment setting script.
1014 def static envScriptCreate(def os, def stepScriptLocation) {
1015     def stepScript = ''
1016     if (os == 'Windows_NT') {
1017         stepScript += "echo Creating TestEnv script\r\n"
1018         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1019
1020         // Create at least an empty script.
1021         stepScript += "echo. > ${stepScriptLocation}\r\n"
1022     }
1023     else {
1024         stepScript += "echo Creating environment setting script\n"
1025         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1026     }
1027
1028     return stepScript
1029 }
1030
1031 // Generates the string for setting stress mode variables.
1032 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1033     def stepScript = ''
1034     if (os == 'Windows_NT') {
1035         stressModeVars.each{ k, v ->
1036             // Write out what we are writing to the script file
1037             stepScript += "echo Setting ${k}=${v}\r\n"
1038             // Write out the set itself to the script file`
1039             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1040         }
1041     }
1042     else {
1043         stressModeVars.each{ k, v ->
1044             // Write out what we are writing to the script file
1045             stepScript += "echo Setting ${k}=${v}\n"
1046             // Write out the set itself to the script file`
1047             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1048         }
1049     }
1050
1051     return stepScript
1052 }
1053
1054 // Append an existing script to an environment script.
1055 // Returns string of commands to do this.
1056 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1057     assert (os == 'Windows_NT')
1058     def stepScript = ''
1059
1060     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1061     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1062
1063     return stepScript
1064 }
1065
1066 // Finalize an environment setting script.
1067 // Returns string of commands to do this.
1068 def static envScriptFinalize(def os, def stepScriptLocation) {
1069     def stepScript = ''
1070
1071     if (os == 'Windows_NT') {
1072         // Display the resulting script. This is useful when looking at the output log file.
1073         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1074         stepScript += "type ${stepScriptLocation}\r\n"
1075     }
1076     else {
1077         stepScript += "chmod +x ${stepScriptLocation}\n"
1078     }
1079
1080     return stepScript
1081 }
1082
1083 def static isNeedDocker(def architecture, def os, def isBuild) {
1084     if (isBuild) {
1085         if (architecture == 'x86' && os == 'Ubuntu') {
1086             return true
1087         }
1088         else if (architecture == 'armem') {
1089             return true
1090         }
1091         else if (architecture == 'arm') {
1092             if (os == 'Ubuntu') {
1093                 return true
1094             }
1095         }
1096         else if (architecture == 'arm64') {
1097             if (os == 'Ubuntu') {
1098                 return true
1099             }
1100         }
1101     }
1102     else {
1103         if (architecture == 'x86' && os == 'Ubuntu') {
1104             return true
1105         }
1106     }
1107     return false
1108 }
1109
1110 def static getDockerImageName(def architecture, def os, def isBuild) {
1111     // We must change some docker private images to official later
1112     if (isBuild) {
1113         if (architecture == 'x86' && os == 'Ubuntu') {
1114             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1115         }
1116         else if (architecture == 'armem') {
1117             if (os == 'Tizen') {
1118                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1119             }
1120         }
1121         else if (architecture == 'arm') {
1122             if (os == 'Ubuntu') {
1123                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1124             }
1125         }
1126         else if (architecture == 'arm64') {
1127             if (os == 'Ubuntu') {
1128                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1129             }
1130         }
1131     }
1132     else {
1133         if (architecture == 'x86' && os == 'Ubuntu') {
1134             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1135         }
1136     }
1137     println("Unknown architecture to use docker: ${architecture} ${os}");
1138     assert false
1139 }
1140
1141
1142 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1143 // and only allowing using this hardware in some specific branches.
1144 def static jobRequiresLimitedHardware(def architecture, def os) {
1145     if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
1146         // These test jobs require ARM64 hardware
1147         return true
1148     }
1149     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1150         // These test jobs require Linux/arm32 hardware
1151         return true
1152     }
1153     else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
1154         // These test jobs require Linux/arm64 hardware
1155         return true
1156     }
1157     else {
1158         return false
1159     }
1160 }
1161
1162 // Calculates the name of the build job based on some typical parameters.
1163 //
1164 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1165     // If the architecture is x64, do not add that info into the build name.
1166     // Need to change around some systems and other builds to pick up the right builds
1167     // to do that.
1168
1169     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1170     if (isBuildOnly) {
1171         suffix += '_bld'
1172     }
1173     def baseName = ''
1174     switch (architecture) {
1175         case 'x64':
1176             if (scenario == 'normal') {
1177                 // For now we leave x64 off of the name for compatibility with other jobs
1178                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1179             }
1180             else if (scenario == 'formatting') {
1181                 // we don't care about the configuration for the formatting job. It runs all configs
1182                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1183             }
1184             else {
1185                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1186             }
1187             break
1188         case 'arm64':
1189             if (os.toLowerCase() == "windows_nt") {
1190                 // These are cross builds
1191                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1192             }
1193             else {
1194                 // Defaults to a small page size set of machines.
1195                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1196             }
1197             break
1198         case 'armem':
1199             // These are cross builds
1200             assert os == 'Tizen'
1201             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1202             break
1203         case 'arm':
1204             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1205             break
1206         case 'x86':
1207         case 'x86_arm_altjit':
1208         case 'x64_arm64_altjit':
1209             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1210             break
1211         default:
1212             println("Unknown architecture: ${architecture}");
1213             assert false
1214             break
1215     }
1216
1217     return baseName + suffix
1218 }
1219
1220 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1221     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1222
1223     // Limited hardware is restricted for non-PR triggers to certain branches.
1224     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1225         return
1226     }
1227
1228     // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
1229     if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1230          return
1231     }
1232
1233     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1234     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1235     if (architecture == 'x86' && os == 'Ubuntu') {
1236         return
1237     }
1238
1239     // Check scenario.
1240     switch (scenario) {
1241         case 'innerloop':
1242         case 'no_tiered_compilation_innerloop':
1243             break
1244         case 'normal':
1245             switch (architecture) {
1246                 case 'x64':
1247                 case 'x86':
1248                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1249                         addPeriodicTriggerHelper(job, '@daily')
1250                     }
1251                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1252                         addGithubPushTriggerHelper(job)
1253                     }
1254                     break
1255                 case 'arm64':
1256                     if (os == 'Windows_NT') {
1257                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1258                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1259                             // Do the builds daily.
1260                             addPeriodicTriggerHelper(job, '@daily')
1261                         }
1262                     }
1263                     else {
1264                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1265                         if (isFlowJob) {
1266                             addPeriodicTriggerHelper(job, '@daily')
1267                         }
1268                     }
1269                     break
1270                 case 'arm':
1271                     if (os == 'Windows_NT') {
1272                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1273                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1274                             // Do the builds daily.
1275                             addPeriodicTriggerHelper(job, '@daily')
1276                         }
1277                     }
1278                     else {
1279                         assert os == 'Ubuntu'
1280                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1281                         if (isFlowJob) {
1282                             // Currently no push triggers, with limited arm Linux hardware.
1283                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1284                             addPeriodicTriggerHelper(job, '@daily')
1285                         }
1286                     }
1287                     break
1288                 case 'armem':
1289                     addGithubPushTriggerHelper(job)
1290                     break
1291                 case 'x86_arm_altjit':
1292                 case 'x64_arm64_altjit':
1293                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1294                     if (configuration == 'Checked') {
1295                         addGithubPushTriggerHelper(job)
1296                     }
1297                     break
1298                 default:
1299                     println("Unknown architecture: ${architecture}");
1300                     assert false
1301                     break
1302             }
1303             break
1304         case 'r2r':
1305             assert !(os in bidailyCrossList)
1306             // r2r gets a push trigger for checked/release
1307             if (configuration == 'Checked' || configuration == 'Release') {
1308                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1309                 if (architecture == 'x64' && os != 'OSX10.12') {
1310                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1311                     if (isFlowJob || os == 'Windows_NT') {
1312                         addGithubPushTriggerHelper(job)
1313                     }
1314                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1315                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1316                     if (isFlowJob) {
1317                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1318                     }
1319                 }
1320                 // For x86, only add per-commit jobs for Windows
1321                 else if (architecture == 'x86') {
1322                     if (os == 'Windows_NT') {
1323                         addGithubPushTriggerHelper(job)
1324                     }
1325                 }
1326                 // arm64 r2r jobs should only run weekly.
1327                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1328                 else if (architecture == 'arm64') {
1329                     if (os == 'Windows_NT') {
1330                         if (isFlowJob) {
1331                             addPeriodicTriggerHelper(job, '@weekly')
1332                         }
1333                     }
1334                 }
1335                 // arm r2r jobs should only run weekly.
1336                 else if (architecture == 'arm') {
1337                     if (isFlowJob) {
1338                         addPeriodicTriggerHelper(job, '@weekly')
1339                     }
1340                 }
1341             }
1342             break
1343         case 'r2r_jitstress1':
1344         case 'r2r_jitstress2':
1345         case 'r2r_jitstress1_tiered':
1346         case 'r2r_jitstress2_tiered':
1347         case 'r2r_jitstressregs1':
1348         case 'r2r_jitstressregs2':
1349         case 'r2r_jitstressregs3':
1350         case 'r2r_jitstressregs4':
1351         case 'r2r_jitstressregs8':
1352         case 'r2r_jitstressregs0x10':
1353         case 'r2r_jitstressregs0x80':
1354         case 'r2r_jitstressregs0x1000':
1355         case 'r2r_jitminopts':
1356         case 'r2r_jitforcerelocs':
1357         case 'r2r_gcstress15':
1358         case 'r2r_no_tiered_compilation':
1359             assert !(os in bidailyCrossList)
1360
1361             // GCStress=C is currently not supported on OS X
1362             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1363                 break
1364             }
1365
1366             // GC Stress 15 r2r gets a push trigger for checked/release
1367             if (configuration == 'Checked' || configuration == 'Release') {
1368                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1369                 if (architecture == 'x64') {
1370                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1371                     if (isFlowJob || os == 'Windows_NT') {
1372                         // Add a weekly periodic trigger
1373                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1374                     }
1375                 }
1376                 // For x86, only add periodic jobs for Windows
1377                 else if (architecture == 'x86') {
1378                     if (os == 'Windows_NT') {
1379                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1380                     }
1381                 }
1382                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1383                 else if (architecture == 'arm64') {
1384                     if (os == 'Windows_NT') {
1385                         if (isFlowJob) {
1386                             addPeriodicTriggerHelper(job, '@weekly')
1387                         }
1388                     }
1389                 }
1390                 else if (architecture == 'arm') {
1391                     if (isFlowJob) {
1392                         addPeriodicTriggerHelper(job, '@weekly')
1393                     }
1394                 }
1395             }
1396             break
1397         case 'longgc':
1398             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1399             assert configuration == 'Release'
1400             assert architecture == 'x64'
1401             addPeriodicTriggerHelper(job, '@daily')
1402             // TODO: Add once external email sending is available again
1403             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1404             break
1405         case 'gcsimulator':
1406             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1407             assert configuration == 'Release'
1408             assert architecture == 'x64'
1409             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1410             // TODO: Add once external email sending is available again
1411             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1412             break
1413         case 'standalone_gc':
1414             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1415             assert (configuration == 'Release' || configuration == 'Checked')
1416             // TODO: Add once external email sending is available again
1417             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1418             addPeriodicTriggerHelper(job, '@daily')
1419             break
1420         case 'gc_reliability_framework':
1421             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1422             assert (configuration == 'Release' || configuration == 'Checked')
1423             // Only triggered by phrase.
1424             break
1425         case 'ilrt':
1426             assert !(os in bidailyCrossList)
1427             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1428             if (architecture == 'x64' && configuration == 'Release') {
1429                 // We don't expect to see a job generated except in these scenarios
1430                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1431                 if (isFlowJob || os == 'Windows_NT') {
1432                     addPeriodicTriggerHelper(job, '@daily')
1433                 }
1434             }
1435             break
1436         case 'jitdiff':
1437             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1438             assert configuration == 'Checked'
1439             assert (architecture == 'x64' || architecture == 'x86')
1440             addGithubPushTriggerHelper(job)
1441             break
1442         case 'formatting':
1443             assert (os == 'Windows_NT' || os == "Ubuntu")
1444             assert architecture == 'x64'
1445             addGithubPushTriggerHelper(job)
1446             break
1447         case 'jitstressregs1':
1448         case 'jitstressregs2':
1449         case 'jitstressregs3':
1450         case 'jitstressregs4':
1451         case 'jitstressregs8':
1452         case 'jitstressregs0x10':
1453         case 'jitstressregs0x80':
1454         case 'jitstressregs0x1000':
1455         case 'minopts':
1456         case 'tieredcompilation':
1457         case 'no_tiered_compilation':
1458         case 'forcerelocs':
1459         case 'jitstress1':
1460         case 'jitstress2':
1461         case 'jitstress1_tiered':
1462         case 'jitstress2_tiered':
1463         case 'jitstress2_jitstressregs1':
1464         case 'jitstress2_jitstressregs2':
1465         case 'jitstress2_jitstressregs3':
1466         case 'jitstress2_jitstressregs4':
1467         case 'jitstress2_jitstressregs8':
1468         case 'jitstress2_jitstressregs0x10':
1469         case 'jitstress2_jitstressregs0x80':
1470         case 'jitstress2_jitstressregs0x1000':
1471         case 'tailcallstress':
1472         case 'jitsse2only':
1473         case 'jitnosimd':
1474         case 'jitnox86hwintrinsic':
1475         case 'jitincompletehwintrinsic':
1476         case 'jitx86hwintrinsicnoavx':
1477         case 'jitx86hwintrinsicnoavx2':
1478         case 'jitx86hwintrinsicnosimd':
1479         case 'corefx_baseline':
1480         case 'corefx_minopts':
1481         case 'corefx_tieredcompilation':
1482         case 'corefx_jitstress1':
1483         case 'corefx_jitstress2':
1484         case 'corefx_jitstressregs1':
1485         case 'corefx_jitstressregs2':
1486         case 'corefx_jitstressregs3':
1487         case 'corefx_jitstressregs4':
1488         case 'corefx_jitstressregs8':
1489         case 'corefx_jitstressregs0x10':
1490         case 'corefx_jitstressregs0x80':
1491         case 'corefx_jitstressregs0x1000':
1492         case 'zapdisable':
1493             if (os == 'CentOS7.1') {
1494                 break
1495             }
1496             if (os in bidailyCrossList) {
1497                 break
1498             }
1499             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1500             // build for the flow job. We don't need cron jobs for these. Note that the
1501             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1502             // function very early, so only non-Windows gets here.
1503             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1504                 break
1505             }
1506             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1507                 break
1508             }
1509             assert (os == 'Windows_NT') || (os in Constants.crossList)
1510             if (jobRequiresLimitedHardware(architecture, os)) {
1511                 addPeriodicTriggerHelper(job, '@weekly')
1512             }
1513             else {
1514                 addPeriodicTriggerHelper(job, '@daily')
1515             }
1516             break
1517         case 'heapverify1':
1518         case 'gcstress0x3':
1519             if (os == 'CentOS7.1') {
1520                 break
1521             }
1522             if (os in bidailyCrossList) {
1523                 break
1524             }
1525             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1526                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1527                 break
1528             }
1529             assert (os == 'Windows_NT') || (os in Constants.crossList)
1530             addPeriodicTriggerHelper(job, '@weekly')
1531             break
1532         case 'gcstress0xc':
1533         case 'gcstress0xc_zapdisable':
1534         case 'gcstress0xc_zapdisable_jitstress2':
1535         case 'gcstress0xc_zapdisable_heapverify1':
1536         case 'gcstress0xc_jitstress1':
1537         case 'gcstress0xc_jitstress2':
1538         case 'gcstress0xc_minopts_heapverify1':
1539             if (os == 'CentOS7.1') {
1540                 break
1541             }
1542             if (os == 'OSX10.12') {
1543                 // GCStress=C is currently not supported on OS X
1544                 break
1545             }
1546             if (os in bidailyCrossList) {
1547                 break
1548             }
1549             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1550                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1551                 break
1552             }
1553             assert (os == 'Windows_NT') || (os in Constants.crossList)
1554             addPeriodicTriggerHelper(job, '@weekly')
1555             break
1556
1557         case 'illink':
1558             // Testing on other operating systems TBD
1559             assert (os == 'Windows_NT' || os == 'Ubuntu')
1560             if (architecture == 'x64' || architecture == 'x86') {
1561                 if (configuration == 'Checked') {
1562                     addPeriodicTriggerHelper(job, '@daily')
1563                 }
1564             }
1565             break
1566
1567         default:
1568             println("Unknown scenario: ${scenario}");
1569             assert false
1570             break
1571     }
1572     return
1573 }
1574
1575 // **************************
1576 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1577 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1578 // builds will do a couple extra steps.
1579 // **************************
1580
1581 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1582 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1583 // it's just used for internal builds
1584 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1585 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1586     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1587
1588     if (isWindowsBuildOnlyJob) {
1589         return
1590     }
1591
1592     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1593     // Non pull request builds.
1594     if (!isPR) {
1595         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1596         return
1597     }
1598
1599      def arm64Users = [
1600         'adityamandaleeka',
1601         'AndyAyersMS',
1602         'briansull',
1603         'BruceForstall',
1604         'CarolEidt',
1605         'cmckinsey',
1606         'echesakovMSFT',
1607         'erozenfeld',
1608         'janvorli',
1609         'jashook',
1610         'JosephTremoulet',
1611         'pgodeq',
1612         'russellhadley',
1613         'RussKeldorph',
1614         'sandreenko',
1615         'sdmaclea',
1616         'swaroop-sridhar',
1617         'jkotas',
1618         'markwilkie',
1619         'weshaggard'
1620     ]
1621
1622     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1623     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1624     def osGroup = getOSGroup(os)
1625     switch (architecture) {
1626         case 'x64': // editor brace matching: {
1627             if (scenario == 'formatting') {
1628                 assert configuration == 'Checked'
1629                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1630                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1631                 }
1632
1633                 break
1634             }
1635
1636             switch (os) {
1637                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1638                 case 'Debian8.4':
1639                 case 'RHEL7.2':
1640                     if (scenario == 'innerloop') {
1641                         assert !isFlowJob
1642                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1643                     }
1644                     else if (scenario == 'normal') {
1645                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1646                     }
1647                     break
1648
1649                 case 'Ubuntu16.04':
1650                     assert !isFlowJob
1651                     assert scenario != 'innerloop'
1652                     // Distinguish with the other architectures (arm and x86)
1653                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1654                     break
1655
1656                 case 'Fedora24':
1657                 case 'Ubuntu16.10':
1658                     assert !isFlowJob
1659                     assert scenario != 'innerloop'
1660                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1661                     break
1662
1663                 case 'Ubuntu':
1664                     if (scenario == 'illink') {
1665                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1666                         break
1667                     }
1668                     else if (scenario == 'corefx_innerloop') {
1669                         if (configuration == 'Checked') {
1670                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests")
1671                         }
1672                         else {
1673                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+CoreFX Tests.*")
1674                         }
1675                         break
1676                     }
1677
1678                     // Triggers on the non-flow jobs aren't necessary here
1679                     // Corefx testing uses non-flow jobs.
1680                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1681                         break
1682                     }
1683
1684                     if (scenario == 'no_tiered_compilation_innerloop') {
1685                         // Default trigger
1686                         if (configuration == 'Checked') {
1687                             def displayStr = getStressModeDisplayName(scenario)
1688                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test (Jit - ${displayStr})")
1689                         }
1690                         break
1691                     }
1692
1693                     // fall through
1694
1695                 case 'OSX10.12':
1696                     // Triggers on the non-flow jobs aren't necessary here
1697                     // Corefx testing uses non-flow jobs.
1698                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1699                         break
1700                     }
1701                     switch (scenario) {
1702                         case 'innerloop':
1703                             // PR Triggered jobs. These jobs will run pri0 tests.
1704                             if (configuration == 'Checked') {
1705                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1706                             }
1707                             break
1708
1709                         case 'normal':
1710                             // OSX uses checked for default PR tests
1711                             if (configuration == 'Checked') {
1712                                 // Default trigger
1713                                 assert !job.name.contains("centos")
1714                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1715                             }
1716                             break
1717
1718                         case 'jitdiff':
1719                             if (configuration == 'Checked') {
1720                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1721                             }
1722                             break
1723
1724                         case 'ilrt':
1725                             if (configuration == 'Release') {
1726                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1727                             }
1728                             break
1729
1730                         case 'longgc':
1731                             if (configuration == 'Release') {
1732                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1733                             }
1734                             break
1735
1736                         case 'gcsimulator':
1737                             if (configuration == 'Release') {
1738                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1739                             }
1740                             break
1741
1742                         case 'standalone_gc':
1743                             if (configuration == 'Release' || configuration == 'Checked') {
1744                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1745                             }
1746                             break
1747
1748                         case 'gc_reliability_framework':
1749                             if (configuration == 'Release' || configuration == 'Checked') {
1750                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1751                             }
1752                             break
1753
1754                         case 'corefx_innerloop':
1755                             if (configuration == 'Checked') {
1756                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests")
1757                             }
1758                             break
1759
1760                         default:
1761                             if (isJitStressScenario(scenario)) {
1762                                 def displayStr = getStressModeDisplayName(scenario)
1763                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1765                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1766                             }
1767                             else if (isR2RScenario(scenario)) {
1768                                 if (configuration == 'Release' || configuration == 'Checked') {
1769                                     def displayStr = getR2RDisplayName(scenario)
1770                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1771                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1772                                 }
1773                             }
1774                             else {
1775                                 println("Unknown scenario: ${scenario}");
1776                                 assert false
1777                             }
1778                             break
1779
1780                     }
1781                     break
1782
1783                 case 'CentOS7.1':
1784                     switch (scenario) {
1785                         case 'innerloop':
1786                             // CentOS uses checked for default PR tests while debug is build only
1787                             if (configuration == 'Debug') {
1788                                 // Default trigger
1789                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1790                             }
1791
1792                             // Make sure this is a flow job to get build and test.
1793                             if (configuration == 'Checked' && isFlowJob) {
1794                                 assert job.name.contains("flow")
1795                                 // Default trigger
1796                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1797                             }
1798                             break
1799
1800                         case 'normal':
1801                             // Make sure this is a flow job to get build and test.
1802                             if (configuration == 'Checked' && isFlowJob) {
1803                                 assert job.name.contains("flow")
1804                                 // Default trigger
1805                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1806                             }
1807                             break
1808
1809                         default:
1810                             if (isR2RScenario(scenario)) {
1811                                 if (configuration == 'Release' || configuration == 'Checked') {
1812                                     def displayStr = getR2RDisplayName(scenario)
1813                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1814                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1815                                 }
1816                             }
1817                             break
1818
1819                     }
1820                     break
1821
1822                 case 'Windows_NT':
1823                     switch (scenario) {
1824                         case 'innerloop':
1825                             // Default trigger
1826                             if (configuration == 'Checked' || configuration == 'Release') {
1827                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1828                             }
1829                             break
1830                         case 'no_tiered_compilation_innerloop':
1831                             // Default trigger
1832                             if (configuration == 'Checked') {
1833                                 def displayStr = getStressModeDisplayName(scenario)
1834                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test (Jit - ${displayStr})")
1835                             }
1836                             break
1837
1838                         case 'normal':
1839                             if (configuration == 'Checked') {
1840                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1841                             }
1842                             break
1843
1844                         case 'jitdiff':
1845                             if (configuration == 'Checked') {
1846                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1847                             }
1848                             break
1849
1850                         case 'ilrt':
1851                             if (configuration == 'Release') {
1852                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1853                             }
1854                             break
1855
1856                         case 'longgc':
1857                             if (configuration == 'Release') {
1858                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1859                             }
1860                             break
1861
1862                         case 'gcsimulator':
1863                             if (configuration == 'Release') {
1864                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1865                             }
1866                             break
1867
1868                         case 'standalone_gc':
1869                             if (configuration == 'Release' || configuration == 'Checked') {
1870                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1871                             }
1872                             break
1873
1874                         case 'gc_reliability_framework':
1875                             if (configuration == 'Release' || configuration == 'Checked') {
1876                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1877                             }
1878                             break
1879
1880                         case 'illink':
1881                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1882                             break
1883                         case 'corefx_innerloop':
1884                             if (configuration == 'Checked') {
1885                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests")
1886                             }
1887                             break
1888
1889                         default:
1890                             if (isJitStressScenario(scenario)) {
1891                                 def displayStr = getStressModeDisplayName(scenario)
1892                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1893                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1894                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1895                             }
1896                             else if (isR2RScenario(scenario)) {
1897                                 if (configuration == 'Release' || configuration == 'Checked') {
1898                                     def displayStr = getR2RDisplayName(scenario)
1899                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1900                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1901                                 }
1902                             }
1903                             else {
1904                                 println("Unknown scenario: ${scenario}");
1905                                 assert false
1906                             }
1907                             break
1908
1909                     }
1910                     break
1911
1912                 default:
1913                     println("Unknown os: ${os}");
1914                     assert false
1915                     break
1916
1917             }
1918
1919             break
1920         // editor brace matching: }
1921
1922         case 'armem': // editor brace matching: {
1923             job.with {
1924                 publishers {
1925                     azureVMAgentPostBuildAction {
1926                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1927                     }
1928                 }
1929             }
1930
1931             switch (os) {
1932                 case 'Tizen':
1933                     architecture = 'armel'
1934                     if (scenario == 'innerloop') {
1935                         if (configuration == 'Checked') {
1936                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1937                         }
1938                     }
1939                     else {
1940                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1941                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1942                     }
1943                     break
1944             }
1945
1946             break
1947         // editor brace matching: }
1948
1949         case 'arm': // editor brace matching: {
1950
1951             // Set up a private trigger
1952             def contextString = "${os} ${architecture} Cross ${configuration}"
1953             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1954             if (scenario == 'innerloop') {
1955                 contextString += " Innerloop"
1956                 triggerString += "\\W+Innerloop"
1957             }
1958             else {
1959                 contextString += " ${scenario}"
1960                 triggerString += "\\W+${scenario}"
1961             }
1962
1963             if (configuration == 'Debug') {
1964                 contextString += " Build"
1965                 triggerString += "\\W+Build"
1966             } else {
1967                 contextString += " Build and Test"
1968                 triggerString += "\\W+Build and Test"
1969             }
1970
1971             triggerString += ".*"
1972
1973             switch (os) {
1974                 case 'Ubuntu':
1975                     // Triggers on the non-flow jobs aren't necessary
1976                     if (!isFlowJob) {
1977                         break
1978                     }
1979
1980                     if (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop') {
1981                         if (configuration == 'Checked') {
1982                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1983                         }
1984                     }
1985                     else {
1986                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1987                     }
1988                     break
1989
1990                 case 'Windows_NT':
1991                     assert isArmWindowsScenario(scenario)
1992
1993                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1994                     // the non-flow Build job. All others need a trigger on the flow job.
1995                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1996                     if (isFlowJob != needsFlowJobTrigger) {
1997                         break
1998                     }
1999
2000                     switch (scenario) {
2001                         case 'innerloop':
2002                             if (configuration == 'Checked') {
2003                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2004                             }
2005                             break
2006                         default:
2007                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2008                             break
2009                     }
2010                     break
2011                 default:
2012                     println("NYI os: ${os}");
2013                     assert false
2014                     break
2015             }
2016             break
2017         // editor brace matching: }
2018         case 'arm64': // editor brace matching: {
2019             // Set up a private trigger
2020             def contextString = "${os} ${architecture} Cross ${configuration}"
2021             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
2022
2023             if (scenario == 'innerloop') {
2024                 contextString += " Innerloop"
2025                 triggerString += "\\W+Innerloop"
2026             }
2027             else {
2028                 contextString += " ${scenario}"
2029                 triggerString += "\\W+${scenario}"
2030             }
2031
2032             if (configuration == 'Debug') {
2033                 contextString += " Build"
2034                 triggerString += "\\W+Build"
2035             } else {
2036                 contextString += " Build and Test"
2037                 triggerString += "\\W+Build and Test"
2038             }
2039
2040             triggerString += ".*"
2041
2042             switch (os) {
2043                 case 'Ubuntu':
2044                     switch (scenario) {
2045                         case 'innerloop':
2046                             if (configuration == 'Debug' && !isFlowJob) {
2047                                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2048                             }
2049                             break
2050                         case 'normal':
2051                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2052                             break
2053                         default:
2054                             if (isR2RScenario(scenario)) {
2055                                 if (configuration == 'Checked' || configuration == 'Release') {
2056                                     def displayStr = getR2RDisplayName(scenario)
2057                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
2058                                 }
2059                             }
2060                             break
2061                     }
2062                     break
2063
2064                 case 'Windows_NT':
2065                     assert isArmWindowsScenario(scenario)
2066
2067                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
2068                     // the non-flow Build job. All others need a trigger on the flow job.
2069                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
2070                     if (isFlowJob != needsFlowJobTrigger) {
2071                         break
2072                     }
2073
2074                     switch (scenario) {
2075                         case 'innerloop':
2076                             if (configuration == 'Checked') {
2077                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2078                             }
2079                             break
2080                         default:
2081                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2082                             break
2083                     }
2084                     break
2085                 default:
2086                     println("NYI os: ${os}");
2087                     assert false
2088                     break
2089             }
2090             break
2091
2092         // editor brace matching: }
2093         case 'x86': // editor brace matching: {
2094             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2095             if (os == 'Ubuntu') {
2096                 // Triggers on the non-flow jobs aren't necessary here
2097                 if (!isFlowJob) {
2098                     break
2099                 }
2100
2101                 // on-demand only for ubuntu x86
2102                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
2103                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
2104                 break
2105
2106             }
2107             switch (scenario) {
2108                 case 'innerloop':
2109                     if (configuration == 'Checked' || configuration == 'Release') {
2110                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
2111                     }
2112                     break
2113
2114                 case 'no_tiered_compilation_innerloop':
2115                     if (configuration == 'Checked') {
2116                         def displayStr = getStressModeDisplayName(scenario)
2117                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test (Jit - ${displayStr})")
2118                     }
2119                     break
2120
2121                 case 'normal':
2122                     if (configuration == 'Checked') {
2123                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
2124                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
2125                     }
2126                     break
2127
2128                 case 'ilrt':
2129                     if (configuration == 'Release') {
2130                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
2131                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2132                     }
2133                     break
2134
2135                 case 'longgc':
2136                     if (configuration == 'Release') {
2137                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
2138                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2139                     }
2140                     break
2141
2142                 case 'gcsimulator':
2143                     if (configuration == 'Release') {
2144                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
2145                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2146                     }
2147                     break
2148
2149                 case 'standalone_gc':
2150                     if (configuration == 'Release' || configuration == 'Checked') {
2151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
2152                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2153                     }
2154                     break
2155
2156                 case 'illink':
2157                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2158                     break
2159
2160                 default:
2161                     if (isJitStressScenario(scenario)) {
2162                         def displayStr = getStressModeDisplayName(scenario)
2163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
2164                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2165                     }
2166                     else if (isR2RScenario(scenario)) {
2167                         if (configuration == 'Release' || configuration == 'Checked') {
2168                             def displayStr = getR2RDisplayName(scenario)
2169                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
2170                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2171                         }
2172                     }
2173                     else {
2174                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
2175                         assert false
2176                     }
2177                     break
2178
2179             }
2180             break
2181
2182          // editor brace matching: }
2183         case 'x64_arm64_altjit':
2184         case 'x86_arm_altjit': // editor brace matching: {
2185             assert (os == 'Windows_NT')
2186             switch (scenario) {
2187                 case 'normal':
2188                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
2189                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
2190                     break
2191                 default:
2192                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
2193                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2194                     break
2195             }
2196             break
2197
2198         // editor brace matching: }
2199         default:
2200             println("Unknown architecture: ${architecture}");
2201             assert false
2202             break
2203     }
2204 }
2205
2206 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2207     def buildCommands = []
2208     def osGroup = getOSGroup(os)
2209     def lowerConfiguration = configuration.toLowerCase()
2210
2211     def priority = '1'
2212     if (isPri0TestScenario(scenario)) {
2213         priority = '0'
2214     }
2215
2216     def doCoreFxTesting = isCoreFxScenario(scenario)
2217
2218     // Calculate the build steps, archival, and xunit results
2219     switch (os) {
2220         case 'Windows_NT': // editor brace matching: {
2221             switch (architecture) {
2222                 case 'x64':
2223                 case 'x86':
2224                 case 'x86_arm_altjit':
2225                 case 'x64_arm64_altjit':
2226                     def arch = architecture
2227                     def buildOpts = ''
2228                     if (architecture == 'x86_arm_altjit') {
2229                         arch = 'x86'
2230                     }
2231                     else if (architecture == 'x64_arm64_altjit') {
2232                         arch = 'x64'
2233                     }
2234
2235                     if (scenario == 'formatting') {
2236                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2237                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2238                         break
2239                     }
2240
2241                     if (scenario == 'illink') {
2242                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2243                     }
2244
2245                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2246                     if ((lowerConfiguration == 'release') &&
2247                         (scenario in Constants.basicScenarios) &&
2248                         (architecture != 'x86_arm_altjit') &&
2249                         (architecture != 'x64_arm64_altjit')) {
2250
2251                         buildOpts += ' -enforcepgo'
2252                     }
2253
2254                     if (doCoreFxTesting) {
2255                         buildOpts += ' skiptests';
2256                     } else {
2257                         buildOpts += " -priority=${priority}"
2258                     }
2259
2260                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2261                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2262                     // 35 characters long.
2263
2264                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2265
2266                     if (!isBuildOnly) {
2267                         def runtestArguments = ''
2268                         def testOpts = 'collectdumps'
2269
2270                         if (isR2RScenario(scenario)) {
2271
2272                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2273                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2274                             // to cause the tests to be crossgen'ed.
2275
2276                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2277                                 testOpts += ' crossgenaltjit protononjit.dll'
2278                             } else {
2279                                 testOpts += ' crossgen'
2280                             }
2281
2282                             testOpts += ' runcrossgentests'
2283                         }
2284                         else if (scenario == 'jitdiff') {
2285                             testOpts += ' jitdisasm crossgen'
2286                         }
2287                         else if (scenario == 'ilrt') {
2288                             testOpts += ' ilasmroundtrip'
2289                         }
2290                         else if (isLongGc(scenario)) {
2291                             testOpts += " ${scenario} sequential"
2292                         }
2293                         else if (scenario == 'standalone_gc') {
2294                             testOpts += ' gcname clrgc.dll'
2295                         }
2296                         else if (scenario == 'illink') {
2297                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2298                         }
2299
2300                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2301                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2302                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2303                         // which is read by the xunit harness.)
2304                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2305                         {
2306                             def timeout = 1800000
2307                             testOpts += " timeout ${timeout}"
2308                         }
2309
2310                         // If we are running a stress mode, we should write out the set of key
2311                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2312
2313                         def envScriptPath = ''
2314                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2315                             def buildCommandsStr = ''
2316                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2317                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2318
2319                             if (isJitStressScenario(scenario)) {
2320                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2321                             }
2322                             else if (isR2RStressScenario(scenario)) {
2323                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2324                             }
2325
2326                             if (architecture == 'x86_arm_altjit') {
2327                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2328                             }
2329                             else if (architecture == 'x64_arm64_altjit') {
2330                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2331                             }
2332
2333                             envScriptFinalize(os, envScriptPath)
2334
2335                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2336                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2337                             // into a single script.
2338                             buildCommands += buildCommandsStr
2339                         }
2340                         else if (architecture == 'x86_arm_altjit') {
2341                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2342                         }
2343                         else if (architecture == 'x64_arm64_altjit') {
2344                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2345                         }
2346                         if (envScriptPath != '') {
2347                             testOpts += " TestEnv ${envScriptPath}"
2348                         }
2349
2350                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2351
2352                         if (doCoreFxTesting) {
2353                             if (scenario == 'corefx_innerloop') {
2354                                 // Create CORE_ROOT and testhost
2355                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2356                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2357
2358                                 // Archive and process (only) the test results
2359                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2360                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2361                             }
2362                             else {
2363                                 def workspaceRelativeFxRoot = "_/fx"
2364                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2365                                 def fxBranch = getFxBranch(branch)
2366
2367                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2368
2369                                 // Archive and process (only) the test results
2370                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2371                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2372
2373                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2374                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2375                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2376                             }
2377                         }
2378                         else if (isGcReliabilityFramework(scenario)) {
2379                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2380                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2381                         }
2382                         else {
2383                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2384                         }
2385                     } // end if (!isBuildOnly)
2386
2387                     if (!doCoreFxTesting) {
2388                         // Run the rest of the build
2389                         // Build the mscorlib for the other OS's
2390                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2391                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2392
2393                         if (arch == 'x64') {
2394                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2395                         }
2396
2397                         // Zip up the tests directory so that we don't use so much space/time copying
2398                         // 10s of thousands of files around.
2399                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2400
2401                         if (!isJitStressScenario(scenario)) {
2402                             // For Windows, pull full test results and test drops for x86/x64.
2403                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2404                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2405                         }
2406
2407                         if (scenario == 'jitdiff') {
2408                             // retrieve jit-dasm output for base commit, and run jit-diff
2409                             if (!isBuildOnly) {
2410                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2411                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2412                             }
2413                         }
2414
2415                         if (!isBuildOnly) {
2416                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2417                         }
2418                     }
2419                     break
2420                 case 'arm':
2421                 case 'arm64':
2422                     assert isArmWindowsScenario(scenario)
2423
2424                     def buildOpts = ''
2425
2426                     if (architecture == 'arm64') {
2427                         buildOpts += " toolset_dir C:\\ats2"
2428                     }
2429
2430                     if (doCoreFxTesting) {
2431                         buildOpts += ' skiptests'
2432                     } else {
2433                         buildOpts += " -priority=${priority}"
2434                     }
2435
2436                     // This is now a build only job. Do not run tests. Use the flow job.
2437                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2438
2439                     if (doCoreFxTesting) {
2440                         assert isBuildOnly
2441
2442                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2443                         def envScriptPath = ''
2444                         def buildCommandsStr = ''
2445                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2446                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2447                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2448                         envScriptFinalize(os, envScriptPath)
2449                         buildCommands += buildCommandsStr
2450
2451                         def workspaceRelativeFxRootLinux = "_/fx"
2452                         def workspaceRelativeFxRootWin = "_\\fx"
2453                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2454                         def fxBranch = getFxBranch(branch)
2455
2456                         def toolsetDirOpt = ''
2457                         if (architecture == 'arm64') {
2458                             toolsetDirOpt = "-toolset_dir C:\\ats2"
2459                         }
2460
2461                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests ${toolsetDirOpt}"
2462
2463                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2464                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2465                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2466
2467                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2468                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2469                     } else {
2470                         // Zip up the tests directory so that we don't use so much space/time copying
2471                         // 10s of thousands of files around.
2472                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2473
2474                         // Add archival.
2475                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2476                     }
2477                     break
2478                 default:
2479                     println("Unknown architecture: ${architecture}");
2480                     assert false
2481                     break
2482             }
2483             break
2484         // end case 'Windows_NT'; editor brace matching: }
2485         case 'Ubuntu':
2486         case 'Ubuntu16.04':
2487         case 'Ubuntu16.10':
2488         case 'Debian8.4':
2489         case 'OSX10.12':
2490         case 'CentOS7.1':
2491         case 'RHEL7.2':
2492         case 'Tizen':
2493         case 'Fedora24': // editor brace matching: {
2494             switch (architecture) {
2495                 case 'x64':
2496                 case 'x86':
2497                     if (architecture == 'x86' && os == 'Ubuntu') {
2498                         // build and PAL test
2499                         def dockerImage = getDockerImageName(architecture, os, true)
2500                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2501                         dockerImage = getDockerImageName(architecture, os, false)
2502                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2503                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2504                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2505                         break
2506                     }
2507
2508                     if (scenario == 'formatting') {
2509                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2510                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2511                         break
2512                     }
2513
2514                     if (scenario == 'illink') {
2515                         assert(os == 'Ubuntu')
2516                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2517                     }
2518
2519                     if (!doCoreFxTesting) {
2520                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2521                         // only on supported OS platforms.
2522                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2523                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2524
2525                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2526                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2527
2528                         // Basic archiving of the build
2529                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2530                         // And pal tests
2531                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2532                     }
2533                     else {
2534                         if (scenario == 'corefx_innerloop') {
2535                             assert os == 'Ubuntu' || 'OSX10.12'
2536                             assert architecture == 'x64'
2537
2538                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2539                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2540                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2541
2542                             break
2543                             // Archive and process (only) the test results
2544                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2545                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2546                         }
2547                         else {
2548                             // Corefx stress testing
2549                             assert os == 'Ubuntu'
2550                             assert architecture == 'x64'
2551                             assert lowerConfiguration == 'checked'
2552                             assert isJitStressScenario(scenario)
2553
2554                             // Build coreclr
2555                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2556
2557                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2558
2559                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2560                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2561                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2562                             buildCommands += envScriptCmds
2563
2564                             // Build and text corefx
2565                             def workspaceRelativeFxRoot = "_/fx"
2566                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2567                             def fxBranch = getFxBranch(branch)
2568
2569                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2570
2571                             // Archive and process (only) the test results
2572                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2573                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2574                         }
2575                     }
2576                     break
2577                 case 'armem':
2578                     // Emulator cross builds for ARM runs on Tizen currently
2579                     assert os == 'Tizen'
2580
2581                     def arm_abi = "armel"
2582                     def linuxCodeName = "tizen"
2583
2584                     // Unzip the Windows test binaries first. Exit with 0
2585                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2586
2587                     // Unpack the corefx binaries
2588                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2589                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2590
2591                     // Call the ARM CI script to cross build and test using docker
2592                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2593                     --mode=docker \\
2594                     --${arm_abi} \\
2595                     --linuxCodeName=${linuxCodeName} \\
2596                     --buildConfig=${lowerConfiguration} \\
2597                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2598                     --coreFxBinDir=./bin/CoreFxBinDir \\
2599                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2600
2601                     // Basic archiving of the build, no pal tests
2602                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2603                     break
2604                 case 'arm64':
2605                 case 'arm':
2606                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2607                     assert (os == 'Ubuntu')
2608
2609                     // Add some useful information to the log file. Ignore return codes.
2610                     buildCommands += "uname -a || true"
2611
2612                     def additionalOpts = ""
2613                     if (architecture == 'arm') {
2614                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2615                     }
2616
2617                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2618                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2619                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2620                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2621                     // ZIP up the generated CoreFX runtime and tests.
2622
2623                     def dockerImage = getDockerImageName(architecture, os, true)
2624                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2625
2626                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2627
2628                     if (doCoreFxTesting) {
2629                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2630
2631                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2632                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2633                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2634                         buildCommands += envScriptCmds
2635
2636                         // Build and text corefx
2637                         def workspaceRelativeFxRootLinux = "_/fx"
2638                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2639                         def fxBranch = getFxBranch(branch)
2640
2641                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2642
2643                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2644                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2645                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2646
2647                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2648                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2649                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2650                     }
2651                     else {
2652                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2653                         // download the appropriate CoreFX packages.
2654                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2655                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2656
2657                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2658
2659                         // ZIP up for the test job (created in the flow job code):
2660                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2661                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2662                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2663                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2664
2665                         // These commands are assumed to be run from the root of the workspace.
2666                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2667                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2668
2669                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2670                     }
2671
2672                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2673                     // the cleanup task in Jenkins can't remove.
2674                     newJob.with {
2675                         publishers {
2676                             azureVMAgentPostBuildAction {
2677                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2678                             }
2679                         }
2680                     }
2681                     break
2682                 default:
2683                     println("Unknown architecture: ${architecture}");
2684                     assert false
2685                     break
2686             }
2687             break
2688         // editor brace matching: }
2689         default:
2690             println("Unknown os: ${os}");
2691             assert false
2692             break
2693     } // os
2694
2695     return buildCommands
2696 }
2697
2698 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2699 // Returns true if the job should be generated.
2700 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2701 {
2702     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2703     if (scenario == 'innerloop' && !isPR) {
2704         return false
2705     }
2706
2707     // Run basic corefx tests only on PR-triggered jobs
2708     // Runs under Release and Checked
2709     if (scenario == 'corefx_innerloop' && !isPR) {
2710         return false
2711     }
2712
2713     // Tizen is only supported for armem architecture
2714     if (os == 'Tizen' && architecture != 'armem') {
2715         return false
2716     }
2717
2718     // Filter based on architecture.
2719
2720     switch (architecture) {
2721         case 'arm64':
2722         case 'arm':
2723             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2724                 return false
2725             }
2726             break
2727         case 'armem':
2728             if (os != 'Tizen') {
2729                 return false
2730             }
2731             break
2732         case 'x86_arm_altjit':
2733         case 'x64_arm64_altjit':
2734             if (os != 'Windows_NT') {
2735                 return false
2736             }
2737             break
2738         case 'x86':
2739             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2740                 return false
2741             }
2742             break
2743         case 'x64':
2744             // Everything implemented
2745             break
2746         default:
2747             println("Unknown architecture: ${architecture}")
2748             assert false
2749             break
2750     }
2751
2752     // Which (Windows) build only jobs are required?
2753
2754     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2755
2756     if (isBuildOnly) {
2757         switch (architecture) {
2758             case 'arm':
2759             case 'arm64':
2760                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2761                 if (!isCoreFxScenario(scenario)) {
2762                     return false
2763                 }
2764                 break
2765             case 'x64':
2766             case 'x86':
2767                 if (!isNormalOrInnerloop) {
2768                     return false
2769                 }
2770                 break
2771             default:
2772                 return false
2773         }
2774     }
2775
2776     // Filter based on scenario.
2777
2778     if (isJitStressScenario(scenario)) {
2779         if (configuration != 'Checked') {
2780             return false
2781         }
2782
2783         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
2784         if (!isEnabledOS) {
2785             return false
2786         }
2787
2788         switch (architecture) {
2789             case 'x64':
2790             case 'x86_arm_altjit':
2791             case 'x64_arm64_altjit':
2792                 break
2793
2794             case 'x86':
2795                 // x86 ubuntu: no stress modes
2796                 if (os == 'Ubuntu') {
2797                     return false
2798                 }
2799                 break
2800
2801             case 'arm':
2802                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2803                 // No "regular" Windows arm corefx jobs, e.g.
2804                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2805                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2806                 if (os == 'Windows_NT') {
2807                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2808                         return false
2809                     }
2810                 }
2811                 else {
2812                     if (!isCoreFxScenario(scenario)) {
2813                         return false
2814                     }
2815                 }
2816                 break
2817
2818             case 'arm64':
2819                 if (os == 'Windows_NT') {
2820                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2821                         return false
2822                     }
2823                 }
2824                 else {
2825                     if (!isCoreFxScenario(scenario)) {
2826                         return false
2827                     }
2828                 }
2829                 break
2830
2831             default:
2832                 // arm64: stress is handled through flow jobs.
2833                 // armem: no stress jobs for ARM emulator.
2834                 return false
2835         }
2836     }
2837     else if (isR2RScenario(scenario)) {
2838         if (os != 'Windows_NT') {
2839             return false
2840         }
2841
2842         if (isR2RBaselineScenario(scenario)) {
2843             // no need for Debug scenario; Checked is sufficient
2844             if (configuration != 'Checked' && configuration != 'Release') {
2845                 return false
2846             }
2847         }
2848         else if (isR2RStressScenario(scenario)) {
2849             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2850             if (configuration != 'Checked') {
2851                 return false
2852             }
2853         }
2854
2855         switch (architecture) {
2856             case 'arm':
2857             case 'arm64':
2858                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2859                 return false
2860
2861             default:
2862                 break
2863         }
2864     }
2865     else {
2866         // Skip scenarios
2867         switch (scenario) {
2868             case 'ilrt':
2869                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2870                 // the default scenario build
2871                 if (os != 'Windows_NT') {
2872                     return false
2873                 }
2874                 // Only x64 for now
2875                 if (architecture != 'x64') {
2876                     return false
2877                 }
2878                 // Release only
2879                 if (configuration != 'Release') {
2880                     return false
2881                 }
2882                 break
2883             case 'jitdiff':
2884                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2885                     return false
2886                 }
2887                 if (architecture != 'x64') {
2888                     return false
2889                 }
2890                 if (configuration != 'Checked') {
2891                     return false
2892                 }
2893                 break
2894             case 'longgc':
2895             case 'gcsimulator':
2896                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2897                     return false
2898                 }
2899                 if (architecture != 'x64') {
2900                     return false
2901                 }
2902                 if (configuration != 'Release') {
2903                     return false
2904                 }
2905                 break
2906             case 'gc_reliability_framework':
2907             case 'standalone_gc':
2908                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2909                     return false
2910                 }
2911
2912                 if (architecture != 'x64') {
2913                     return false
2914                 }
2915
2916                 if (configuration != 'Release' && configuration != 'Checked') {
2917                     return false
2918                 }
2919                 break
2920             // We only run Windows and Ubuntu x64 Checked for formatting right now
2921             case 'formatting':
2922                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2923                     return false
2924                 }
2925                 if (architecture != 'x64') {
2926                     return false
2927                 }
2928                 if (configuration != 'Checked') {
2929                     return false
2930                 }
2931                 break
2932             case 'illink':
2933                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2934                     return false
2935                 }
2936                 if (architecture != 'x64' && architecture != 'x86') {
2937                     return false
2938                 }
2939                 break
2940             case 'normal':
2941                 // Nothing skipped
2942                 break
2943             case 'innerloop':
2944                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2945                     return false
2946                 }
2947                 break
2948             case 'corefx_innerloop':
2949                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2950                     return false
2951                 }
2952                 if (architecture != 'x64') {
2953                     return false
2954                 }
2955                 break
2956             default:
2957                 println("Unknown scenario: ${scenario}")
2958                 assert false
2959                 break
2960         }
2961     }
2962
2963     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2964     // runtime behavior, not compile-time behavior, are not interesting.
2965     switch (architecture) {
2966         case 'x86_arm_altjit':
2967         case 'x64_arm64_altjit':
2968             if (isGCStressRelatedTesting(scenario)) {
2969                 return false
2970             }
2971             break
2972         default:
2973             break
2974     }
2975
2976     // The job was not filtered out, so we should generate it!
2977     return true
2978 }
2979
2980 Constants.allScenarios.each { scenario ->
2981     [true, false].each { isPR ->
2982         Constants.architectureList.each { architecture ->
2983             Constants.configurationList.each { configuration ->
2984                 Constants.osList.each { os ->
2985                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2986                     // and reset the os to Windows_NT
2987                     def isBuildOnly = false
2988                     if (os == 'Windows_NT_BuildOnly') {
2989                         isBuildOnly = true
2990                         os = 'Windows_NT'
2991                     }
2992
2993                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2994                         return
2995                     }
2996
2997                     // Calculate names
2998                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2999                     def folderName = getJobFolder(scenario)
3000
3001                     // Create the new job
3002                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
3003
3004                     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
3005                     // "top-level" jobs. Build only jobs are such jobs.
3006                     if (!isBuildOnly)
3007                     {
3008                         addToViews(newJob, isPR, architecture, os)
3009                     }
3010
3011                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
3012
3013                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3014                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
3015                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
3016
3017                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
3018                     // We don't use a flow job for this, but we do depend on there being existing builds with these
3019                     // artifacts produced.
3020                     if ((architecture == 'armem') && (os == 'Tizen')) {
3021                         // Define the Windows Tests and Corefx build job names
3022                         def lowerConfiguration = configuration.toLowerCase()
3023                         def WindowsTestsName = projectFolder + '/' +
3024                                                Utilities.getFullJobName(project,
3025                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
3026                                                                         false)
3027                         def fxBranch = getFxBranch(branch)
3028                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
3029                                            Utilities.getFolderName(fxBranch)
3030
3031                         def arm_abi = 'armel'
3032                         def corefx_os = 'tizen'
3033
3034                         // Let's use release CoreFX to test checked CoreCLR,
3035                         // because we do not generate checked CoreFX in CoreFX CI yet.
3036                         def corefx_lowerConfiguration = lowerConfiguration
3037                         if (lowerConfiguration == 'checked') {
3038                             corefx_lowerConfiguration = 'release'
3039                         }
3040
3041                         // Copy the Windows test binaries and the Corefx build binaries
3042                         newJob.with {
3043                             steps {
3044                                 copyArtifacts(WindowsTestsName) {
3045                                     includePatterns('bin/tests/tests.zip')
3046                                     buildSelector {
3047                                         latestSuccessful(true)
3048                                     }
3049                                 }
3050                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
3051                                     includePatterns('bin/build.tar.gz')
3052                                     buildSelector {
3053                                         latestSuccessful(true)
3054                                     }
3055                                 }
3056                             } // steps
3057                         } // newJob.with
3058                     }
3059
3060                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
3061
3062                     newJob.with {
3063                         steps {
3064                             if (os == 'Windows_NT') {
3065                                 buildCommands.each { buildCommand ->
3066                                     batchFile(buildCommand)
3067                                 }
3068                             }
3069                             else {
3070                                 buildCommands.each { buildCommand ->
3071                                     shell(buildCommand)
3072                                 }
3073                             }
3074                         } // steps
3075                     } // newJob.with
3076
3077                 } // os
3078             } // configuration
3079         } // architecture
3080     } // isPR
3081 } // scenario
3082
3083 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
3084 // Returns the newly created job.
3085 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3086 {
3087     def osGroup = getOSGroup(os)
3088     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3089
3090     def jobFolder = getJobFolder(scenario)
3091     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3092         parameters {
3093             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3094         }
3095
3096         steps {
3097             // Set up the copies
3098
3099             // Coreclr build we are trying to test
3100             //
3101             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3102
3103             copyArtifacts(inputCoreCLRBuildName) {
3104                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3105                 buildSelector {
3106                     buildNumber('${CORECLR_BUILD}')
3107                 }
3108             }
3109
3110             if (isCoreFxScenario(scenario)) {
3111
3112                 // Only arm/arm64 supported for corefx testing now.
3113                 assert architecture == 'arm' || architecture == 'arm64'
3114
3115                 // Unzip CoreFx runtime
3116                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3117
3118                 // Unzip CoreFx tests.
3119                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3120
3121                 // Add the script to run the corefx tests
3122                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3123                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
3124                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3125                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3126
3127             } else { // !isCoreFxScenario(scenario)
3128
3129                 // Unzip tests.
3130                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3131
3132                 def buildCommands = ""
3133
3134                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3135                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3136                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3137
3138                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3139                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3140
3141                 // For all jobs
3142                 addEnvVariable("CORE_ROOT", coreRootLocation)
3143                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3144                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3145
3146                 // If we are running a stress mode, we'll set those variables as well
3147                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3148                     def stressValues = null
3149                     if (isJitStressScenario(scenario)) {
3150                         stressValues = Constants.jitStressModeScenarios[scenario]
3151                     }
3152                     else {
3153                         stressValues = Constants.r2rStressScenarios[scenario]
3154                     }
3155
3156                     stressValues.each { key, value ->
3157                         addEnvVariable(key, value)
3158                     }
3159                 }
3160
3161                 if (isR2RScenario(scenario)) {
3162                     // Crossgen the framework assemblies.
3163                     buildCommands += """
3164 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3165 @goto skip_PrecompileAssembly
3166
3167 :PrecompileAssembly
3168 @REM Skip mscorlib since it is already precompiled.
3169 @if /I "%3" == "mscorlib.dll" exit /b 0
3170 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3171
3172 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3173 @if "%errorlevel%" == "-2146230517" (
3174     echo %2 is not a managed assembly.
3175 ) else if "%errorlevel%" == "-2146234344" (
3176     echo %2 is not a managed assembly.
3177 ) else if %errorlevel% neq 0 (
3178     echo Unable to precompile %2
3179 ) else (
3180     echo Precompiled %2
3181 )
3182 @exit /b 0
3183
3184 :skip_PrecompileAssembly
3185 """
3186
3187                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3188                     // crossgen on tests before running them.
3189                     addEnvVariable("RunCrossGen", "true")
3190                 } // isR2RScenario(scenario)
3191
3192                 // Create the smarty command
3193                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3194                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3195                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3196                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3197
3198                 // Exclude tests based on scenario.
3199                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3200                     addArchSpecificExclude(architecture, excludeTag)
3201                 }
3202
3203                 if (isPri0TestScenario(scenario)) {
3204                     addExclude("pri1")
3205                 }
3206
3207                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3208                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3209                 addExclude("LONG_RUNNING")
3210
3211                 smartyCommand += "/lstFile Tests.lst"
3212
3213                 def testListArch = [
3214                     'arm64': 'arm64',
3215                     'arm': 'arm'
3216                 ]
3217
3218                 def archLocation = testListArch[architecture]
3219
3220                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3221
3222                 if (architecture == "arm64") {
3223                     addCommand("copy C:\\Jenkins\\vcruntime140.dll bin\\tests\\${osGroup}.${architecture}.${configuration}\\Tests\\Core_Root")
3224                     addCommand("copy C:\\Jenkins\\vcruntime140d.dll bin\\tests\\${osGroup}.${architecture}.${configuration}\\Tests\\Core_Root")
3225                 }
3226
3227                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3228                 addCommand("${smartyCommand}")
3229
3230                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3231                 // However, we also need to remove all the variables that were set during this batch file, so we
3232                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3233                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3234                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3235                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3236                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3237                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3238
3239                 // ZIP up the smarty output, no matter what the smarty result.
3240                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3241
3242                 addCommand("echo %errorlevel%")
3243                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3244
3245                 // Use the smarty errorlevel as the script errorlevel.
3246                 addCommand("exit /b %__save_smarty_errorlevel%")
3247
3248                 batchFile(buildCommands)
3249             } // non-corefx testing
3250         } // steps
3251     } // job
3252
3253     if (!isCoreFxScenario(scenario)) {
3254         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3255
3256         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3257         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3258         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3259         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3260     }
3261
3262     return newJob
3263 }
3264
3265 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3266 // E.g., non-Windows tests.
3267 // Returns the newly created job.
3268 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3269 {
3270     def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
3271     def doCoreFxTesting = isCoreFxScenario(scenario)
3272
3273     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3274
3275     def osGroup = getOSGroup(os)
3276     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3277
3278     def testOpts = ''
3279     def useServerGC = false
3280
3281     // Enable Server GC for Ubuntu PR builds
3282     // REVIEW: why? Does this apply to all architectures? Why only PR?
3283     if (os == 'Ubuntu' && isPR) {
3284         testOpts += ' --useServerGC'
3285         useServerGC = true
3286     }
3287
3288     if (isR2RScenario(scenario)) {
3289
3290         testOpts += ' --crossgen --runcrossgentests'
3291
3292         if (scenario == 'r2r_jitstress1') {
3293             testOpts += ' --jitstress=1'
3294         }
3295         else if (scenario == 'r2r_jitstress2') {
3296             testOpts += ' --jitstress=2'
3297         }
3298         else if (scenario == 'r2r_jitstress1_tiered') {
3299             testOpts += ' --jitstress=1'
3300         }
3301         else if (scenario == 'r2r_jitstress2_tiered') {
3302             testOpts += ' --jitstress=2'
3303         }
3304         else if (scenario == 'r2r_jitstressregs1') {
3305             testOpts += ' --jitstressregs=1'
3306         }
3307         else if (scenario == 'r2r_jitstressregs2') {
3308             testOpts += ' --jitstressregs=2'
3309         }
3310         else if (scenario == 'r2r_jitstressregs3') {
3311             testOpts += ' --jitstressregs=3'
3312         }
3313         else if (scenario == 'r2r_jitstressregs4') {
3314             testOpts += ' --jitstressregs=4'
3315         }
3316         else if (scenario == 'r2r_jitstressregs8') {
3317             testOpts += ' --jitstressregs=8'
3318         }
3319         else if (scenario == 'r2r_jitstressregs0x10') {
3320             testOpts += ' --jitstressregs=0x10'
3321         }
3322         else if (scenario == 'r2r_jitstressregs0x80') {
3323             testOpts += ' --jitstressregs=0x80'
3324         }
3325         else if (scenario == 'r2r_jitstressregs0x1000') {
3326             testOpts += ' --jitstressregs=0x1000'
3327         }
3328         else if (scenario == 'r2r_jitminopts') {
3329             testOpts += ' --jitminopts'
3330         }
3331         else if (scenario == 'r2r_jitforcerelocs') {
3332             testOpts += ' --jitforcerelocs'
3333         }
3334         else if (scenario == 'r2r_gcstress15') {
3335             testOpts += ' --gcstresslevel=0xF'
3336         }
3337     }
3338     else if (scenario == 'jitdiff') {
3339         testOpts += ' --jitdisasm --crossgen'
3340     }
3341     else if (scenario == 'illink') {
3342         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3343     }
3344     else if (isLongGc(scenario)) {
3345         // Long GC tests behave very poorly when they are not
3346         // the only test running (many of them allocate until OOM).
3347         testOpts += ' --sequential'
3348
3349         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3350         // for running long GC and GCSimulator tests, respectively. We don't use them
3351         // here because using a playlist file produces much more readable output on the CI machines
3352         // and reduces running time.
3353         //
3354         // The Long GC playlist contains all of the tests that are
3355         // going to be run. The GCSimulator playlist contains all of
3356         // the GC simulator tests.
3357         if (scenario == 'longgc') {
3358             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3359         }
3360         else if (scenario == 'gcsimulator') {
3361             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3362         }
3363     }
3364     else if (isGcReliabilityFramework(scenario)) {
3365         testOpts += ' --build-overlay-only'
3366     }
3367     else if (scenario == 'standalone_gc') {
3368         if (osGroup == 'OSX') {
3369             testOpts += ' --gcname=libclrgc.dylib'
3370         }
3371         else if (osGroup == 'Linux') {
3372             testOpts += ' --gcname=libclrgc.so'
3373         }
3374         else {
3375             println("Unexpected OS group: ${osGroup} for os ${os}")
3376             assert false
3377         }
3378     }
3379
3380     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3381     // will be null in this case.
3382
3383     def jobFolder = getJobFolder(scenario)
3384     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3385         parameters {
3386             if (inputTestsBuildName != null) {
3387                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3388             }
3389             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3390         }
3391
3392         steps {
3393             // Set up the copies
3394
3395             // Coreclr build containing the tests and mscorlib
3396             // pri1 jobs still need to copy windows_nt built tests
3397             if (inputTestsBuildName != null) {
3398                 copyArtifacts(inputTestsBuildName) {
3399                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3400                     buildSelector {
3401                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3402                     }
3403                 }
3404             }
3405
3406             // Coreclr build we are trying to test
3407             //
3408             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3409
3410             copyArtifacts(inputCoreCLRBuildName) {
3411                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3412                 buildSelector {
3413                     buildNumber('${CORECLR_BUILD}')
3414                 }
3415             }
3416
3417             if (isUbuntuArmJob) {
3418                 // Add some useful information to the log file. Ignore return codes.
3419                 shell("uname -a || true")
3420             }
3421
3422             if (architecture == 'x86') {
3423                 shell("mkdir ./bin/CoreFxNative")
3424
3425                 def fxBranch = getFxBranch(branch)
3426                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3427
3428                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3429                     includePatterns('bin/build.tar.gz')
3430                     targetDirectory('bin/CoreFxNative')
3431                     buildSelector {
3432                         latestSuccessful(true)
3433                     }
3434                 }
3435
3436                 shell("mkdir ./bin/CoreFxBinDir")
3437                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3438             }
3439
3440             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3441             if (doCoreFxTesting) {
3442                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3443                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3444             }
3445             else {
3446                 // Unzip the tests first.  Exit with 0
3447                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3448                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3449             }
3450
3451             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3452             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3453             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3454             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3455             // copied correctly.
3456             if (!doCoreFxTesting) {
3457                 if (isUbuntuArmJob) {
3458                     def lowerConfiguration = configuration.toLowerCase()
3459                     shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3460                     shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3461                 }
3462                 else {
3463                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3464                 }
3465             }
3466
3467             // Execute the tests
3468             def runDocker = isNeedDocker(architecture, os, false)
3469             def dockerPrefix = ""
3470             def dockerCmd = ""
3471             if (runDocker) {
3472                 def dockerImage = getDockerImageName(architecture, os, false)
3473                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3474                 dockerCmd = dockerPrefix + "${dockerImage} "
3475             }
3476
3477             // If we are running a stress mode, we'll set those variables first.
3478             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3479             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3480                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3481                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3482                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3483                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3484                 shell("${envScriptCmds}")
3485                 testOpts += " --test-env=${scriptFileName}"
3486             }
3487
3488             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3489             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3490             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3491             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3492             if (isGCStressRelatedTesting(scenario)) {
3493                 if (architecture == 'x64') {
3494                     shell('./init-tools.sh')
3495                 }
3496             }
3497
3498             if (doCoreFxTesting) {
3499                 shell("""\
3500 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3501             }
3502             else {
3503                 def runScript = "${dockerCmd}./tests/runtest.sh"
3504
3505                 shell("""\
3506 ${runScript} \\
3507     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3508     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3509     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3510     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3511             }
3512
3513             if (isGcReliabilityFramework(scenario)) {
3514                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3515                 if (useServerGC) {
3516                     if (runDocker) {
3517                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3518                     }
3519                     else {
3520                         shell("export COMPlus_gcServer=1")
3521                     }
3522                 }
3523
3524                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3525             }
3526         } // steps
3527     } // job
3528
3529     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3530     if (os in ['Ubuntu']) {
3531         SummaryBuilder summaries = new SummaryBuilder()
3532         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3533         summaries.emit(newJob)
3534     }
3535
3536     if (doCoreFxTesting) {
3537         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3538         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3539     }
3540     else {
3541         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3542         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3543     }
3544
3545     return newJob
3546 }
3547
3548 // Create a test job that will be used by a flow job.
3549 // Returns the newly created job.
3550 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3551 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3552 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3553 {
3554     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3555
3556     def newJob = null
3557     if (windowsArmJob) {
3558         assert inputTestsBuildName == null
3559         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3560     } else {
3561         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3562     }
3563
3564     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3565
3566     if (scenario == 'jitdiff') {
3567         def osGroup = getOSGroup(os)
3568         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3569     }
3570
3571     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3572     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3573
3574     return newJob
3575 }
3576
3577 // Create a flow job to tie together a build job with the given test job.
3578 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3579 // Returns the new flow job.
3580 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3581 {
3582     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3583     // Linux CoreCLR test
3584     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3585     def jobFolder = getJobFolder(scenario)
3586
3587     def newFlowJob = null
3588
3589     if (inputTestsBuildName == null) {
3590         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3591                         buildFlow("""\
3592 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3593
3594 // And then build the test build
3595 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3596 """)
3597         }
3598         JobReport.Report.addReference(inputCoreCLRBuildName)
3599         JobReport.Report.addReference(fullTestJobName)
3600     }
3601     else {
3602         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3603                         buildFlow("""\
3604 // Build the input jobs in parallel
3605 parallel (
3606 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3607 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3608 )
3609
3610 // And then build the test build
3611 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3612                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3613 """)
3614         }
3615         JobReport.Report.addReference(inputCoreCLRBuildName)
3616         JobReport.Report.addReference(inputTestsBuildName)
3617         JobReport.Report.addReference(fullTestJobName)
3618     }
3619
3620     addToViews(newFlowJob, isPR, architecture, os)
3621
3622     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3623
3624     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3625     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3626
3627     return newFlowJob
3628 }
3629
3630 // Determine if we should generate a flow job for the given parameters.
3631 // Returns true if the job should be generated.
3632 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3633 {
3634     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3635     if (scenario == 'innerloop' && !isPR) {
3636         return false
3637     }
3638
3639     if (scenario == 'corefx_innerloop') {
3640         return false
3641     }
3642
3643     // Filter based on OS and architecture.
3644
3645     switch (architecture) {
3646         case 'arm64':
3647         case 'arm':
3648             if (os != "Ubuntu" && os != "Windows_NT") {
3649                 return false
3650             }
3651             break
3652         case 'x86':
3653             if (os != "Ubuntu") {
3654                 return false
3655             }
3656             break
3657         case 'x64':
3658             if (!(os in Constants.crossList)) {
3659                 return false
3660             }
3661             if (os == "Windows_NT") {
3662                 return false
3663             }
3664             break
3665         case 'armem':
3666         case 'x86_arm_altjit':
3667         case 'x64_arm64_altjit':
3668             // No flow jobs
3669             return false
3670         default:
3671             println("Unknown architecture: ${architecture}")
3672             assert false
3673             break
3674     }
3675
3676     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3677
3678     // Filter based on scenario in OS.
3679
3680     if (os == 'Windows_NT') {
3681         assert architecture == 'arm' || architecture == 'arm64'
3682         if (!isArmWindowsScenario(scenario)) {
3683             return false
3684         }
3685         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3686             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3687             // To do that, it doesn't require a flow job.
3688             return false
3689         }
3690     }
3691     else {
3692         // Non-Windows
3693         if (architecture == 'arm64') {
3694             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3695                 return false
3696             }
3697         }
3698         else if (architecture == 'arm') {
3699             if (!(scenario in Constants.validLinuxArmScenarios)) {
3700                 return false
3701             }
3702         }
3703         else if (architecture == 'x86') {
3704             // Linux/x86 only want innerloop and default test
3705             if (!isNormalOrInnerloop) {
3706                 return false
3707             }
3708         }
3709         else if (architecture == 'x64') {
3710             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3711             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3712             // build from a Windows machine.
3713             if (isCoreFxScenario(scenario)) {
3714                 return false
3715             }
3716         }
3717     }
3718
3719     // For CentOS, we only want Checked/Release builds.
3720     if (os == 'CentOS7.1') {
3721         if (configuration != 'Checked' && configuration != 'Release') {
3722             return false
3723         }
3724         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3725             return false
3726         }
3727     }
3728
3729     // For RedHat and Debian, we only do Release builds.
3730     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3731         if (configuration != 'Release') {
3732             return false
3733         }
3734         if (!isNormalOrInnerloop) {
3735             return false
3736         }
3737     }
3738
3739     // Next, filter based on scenario.
3740
3741     if (isJitStressScenario(scenario)) {
3742         if (configuration != 'Checked') {
3743             return false
3744         }
3745     }
3746     else if (isR2RBaselineScenario(scenario)) {
3747         if (configuration != 'Checked' && configuration != 'Release') {
3748             return false
3749         }
3750     }
3751     else if (isR2RStressScenario(scenario)) {
3752         if (configuration != 'Checked') {
3753             return false
3754         }
3755     }
3756     else {
3757         // Skip scenarios
3758         switch (scenario) {
3759             case 'ilrt':
3760             case 'longgc':
3761             case 'gcsimulator':
3762                 // Long GC tests take a long time on non-Release builds
3763                 // ilrt is also Release only
3764                 if (configuration != 'Release') {
3765                     return false
3766                 }
3767                 break
3768
3769             case 'jitdiff':
3770                 if (configuration != 'Checked') {
3771                     return false
3772                 }
3773                 break
3774
3775             case 'gc_reliability_framework':
3776             case 'standalone_gc':
3777                 if (configuration != 'Release' && configuration != 'Checked') {
3778                     return false
3779                 }
3780                 break
3781
3782             case 'formatting':
3783                 return false
3784
3785             case 'illink':
3786                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3787                     return false
3788                 }
3789                 break
3790
3791             case 'normal':
3792                 // Nothing skipped
3793                 break
3794
3795             case 'innerloop':
3796                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3797                     return false
3798                 }
3799                 break
3800
3801             default:
3802                 println("Unknown scenario: ${scenario}")
3803                 assert false
3804                 break
3805         }
3806     }
3807
3808     // The job was not filtered out, so we should generate it!
3809     return true
3810 }
3811
3812 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3813 Constants.allScenarios.each { scenario ->
3814     [true, false].each { isPR ->
3815         Constants.architectureList.each { architecture ->
3816             Constants.configurationList.each { configuration ->
3817                 Constants.osList.each { os ->
3818
3819                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3820                         return
3821                     }
3822
3823                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3824                     def doCoreFxTesting = isCoreFxScenario(scenario)
3825
3826                     // Figure out the job name of the CoreCLR build the test will depend on.
3827
3828                     def inputCoreCLRBuildScenario = isPri0TestScenario(scenario) ? 'innerloop' : 'normal'
3829                     def inputCoreCLRBuildIsBuildOnly = false
3830                     if (doCoreFxTesting) {
3831                         // Every CoreFx test depends on its own unique build.
3832                         inputCoreCLRBuildScenario = scenario
3833                         if (windowsArmJob) {
3834                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3835                             inputCoreCLRBuildIsBuildOnly = true
3836                         }
3837                     }
3838                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3839                     def inputCoreCLRBuildName = projectFolder + '/' +
3840                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3841
3842                     // Figure out the name of the build job that the test job will depend on.
3843                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3844                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3845                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3846                     // CoreFX build creates the tests.
3847
3848                     def inputTestsBuildName = null
3849
3850                     if (!windowsArmJob && !doCoreFxTesting) {
3851                         def testBuildScenario = isPri0TestScenario(scenario) ? 'innerloop' : 'normal'
3852
3853                         def inputTestsBuildArch = architecture
3854                         if (architecture == "arm64") {
3855                             // Use the x64 test build for arm64 unix
3856                             inputTestsBuildArch = "x64"
3857                         }
3858                         else if (architecture == "arm") {
3859                             // Use the x86 test build for arm unix
3860                             inputTestsBuildArch = "x86"
3861                         }
3862
3863                         def inputTestsBuildIsBuildOnly = true
3864
3865                         inputTestsBuildName = projectFolder + '/' +
3866                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3867                     }
3868
3869                     // =============================================================================================
3870                     // Create the test job
3871                     // =============================================================================================
3872
3873                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3874
3875                     // =============================================================================================
3876                     // Create a build flow to join together the build and tests required to run this test.
3877                     // =============================================================================================
3878
3879                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3880                         // Do not create the flow job for RHEL jobs.
3881                         return
3882                     }
3883
3884                     def fullTestJobName = projectFolder + '/' + testJob.name
3885                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3886
3887                 } // os
3888             } // configuration
3889         } // architecture
3890     } // isPR
3891 } // scenario
3892
3893 JobReport.Report.generateJobReport(out)
3894
3895 // Make the call to generate the help job
3896 Utilities.createHelperJob(this, project, branch,
3897     "Welcome to the ${project} Repository",  // This is prepended to the help message
3898     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3899
3900 Utilities.addCROSSCheck(this, project, branch)