netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  39     def static LimitedHardwareBranches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  73                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  74                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  75                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  76                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  77                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  78                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  79                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  80                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  81                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  82                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  83                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  84                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  85                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  86                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
  87                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
  88                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  89                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  90                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  91                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  92                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  93                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  94                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  95                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  96                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
  97                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  98                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
  99                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 100                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 101                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 102                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 103                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 104                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 105                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 106                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 107                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 108                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 109                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 110                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 111                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 112                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 113                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 114                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 115                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 116                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 117                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 118                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 119                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 120                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 121                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 122                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 123                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 124                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 125                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 126                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 127     ]
 128
 129     // This is a set of ReadyToRun stress scenarios
 130     def static r2rStressScenarios = [
 131                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 132                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 133                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 134                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 135                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 136                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 137                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 138                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 139                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 140                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 141                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 142                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 143                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 144                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 145                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 146                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 147     ]
 148
 149     // This is the basic set of scenarios
 150     def static basicScenarios = [
 151                'innerloop',
 152                'normal',
 153                'ilrt',
 154                'r2r',
 155                'longgc',
 156                'formatting',
 157                'gcsimulator',
 158                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 159                'standalone_gc',
 160                'gc_reliability_framework',
 161                'illink',
 162                'corefx_innerloop']
 163
 164     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 165
 166     // Valid PR trigger combinations.
 167     def static prTriggeredValidInnerLoopCombos = [
 168         'Windows_NT': [
 169             'x64': [
 170                 'Checked'
 171             ],
 172             'x86': [
 173                 'Checked',
 174                 'Release'
 175             ],
 176             'arm': [
 177                 'Debug',
 178                 'Checked'
 179             ],
 180             'arm64': [
 181                 'Debug',
 182                 'Checked'
 183             ]
 184         ],
 185         'Windows_NT_BuildOnly': [
 186             'x64': [
 187                 'Checked',
 188                 'Release'
 189             ],
 190             'x86': [
 191                 'Checked',
 192                 'Release'
 193             ],
 194             'arm': [
 195                 'Checked'
 196             ],
 197         ],
 198         'Ubuntu': [
 199             'x64': [
 200                 'Checked'
 201             ],
 202             'arm64': [
 203                 'Debug'
 204             ],
 205             'arm': [
 206                 'Checked'
 207             ]
 208         ],
 209         'CentOS7.1': [
 210             'x64': [
 211                 'Debug',
 212                 'Checked'
 213             ]
 214         ],
 215         'OSX10.12': [
 216             'x64': [
 217                 'Checked'
 218             ]
 219         ],
 220         'Tizen': [
 221             'armem': [
 222                 'Checked'
 223             ]
 224         ]
 225     ]
 226
 227     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 228     // to Tests.lst file categories to exclude.
 229     //
 230     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 231     // and with the same values, with some commented out, for easier maintenance.
 232     //
 233     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 234     //
 235     def static validArmWindowsScenarios = [
 236                'innerloop':                              [],
 237                'normal':                                 [],
 238                // 'ilrt'
 239                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 240                // 'longgc'
 241                // 'formatting'
 242                // 'gcsimulator'
 243                // 'jitdiff'
 244                // 'standalone_gc'
 245                // 'gc_reliability_framework'
 246                // 'illink'
 247                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 254                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 260                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 262                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 263                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 264                'tieredcompilation':                      [],
 265                'no_tiered_compilation':                  [],
 266                'no_tiered_compilation_innerloop':        [],
 267                'forcerelocs':                            [],
 268                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 277                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 278                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 279                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 280                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 281                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 282                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 283                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 284                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 285                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 289                // 'jitsse2only'                          // Only relevant to xarch
 290                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 291                // 'jitincompletehwintrinsic'
 292                // 'jitx86hwintrinsicnoavx'
 293                // 'jitx86hwintrinsicnoavx2'
 294                // 'jitx86hwintrinsicnosimd'
 295                // 'jitnox86hwintrinsic'
 296                'corefx_baseline':                        [],    // corefx tests don't use smarty
 297                'corefx_minopts':                         [],    // corefx tests don't use smarty
 298                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 299                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 300                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 301                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 302                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 303                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 304                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 305                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 306                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 307                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 308                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 309                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 310                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 311                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 312                'heapverify1':                            [],
 313                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 314                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 316                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 319
 320                //
 321                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 322                //
 323
 324                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 325                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 326                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 327                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 328                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 329                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 330                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 331                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 332                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 333                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 334                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 336                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 337                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 338                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 339                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 340                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 341                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 342                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 343     ]
 344
 345     def static validLinuxArmScenarios = [
 346                'innerloop',
 347                'normal',
 348                // 'ilrt'
 349                'r2r',
 350                // 'longgc'
 351                // 'formatting'
 352                // 'gcsimulator'
 353                // 'jitdiff'
 354                // 'standalone_gc'
 355                // 'gc_reliability_framework'
 356                // 'illink'
 357                'r2r_jitstress1',
 358                'r2r_jitstress2',
 359                'r2r_jitstress1_tiered',
 360                'r2r_jitstress2_tiered',
 361                'r2r_jitstressregs1',
 362                'r2r_jitstressregs2',
 363                'r2r_jitstressregs3',
 364                'r2r_jitstressregs4',
 365                'r2r_jitstressregs8',
 366                'r2r_jitstressregs0x10',
 367                'r2r_jitstressregs0x80',
 368                'r2r_jitstressregs0x1000',
 369                'r2r_jitminopts',
 370                'r2r_jitforcerelocs',
 371                'r2r_gcstress15',
 372                'r2r_no_tiered_compilation',
 373                'minopts',
 374                'tieredcompilation',
 375                'no_tiered_compilation',
 376                'no_tiered_compilation_innerloop',
 377                'forcerelocs',
 378                'jitstress1',
 379                'jitstress2',
 380                'jitstress1_tiered',
 381                'jitstress2_tiered',
 382                'jitstressregs1',
 383                'jitstressregs2',
 384                'jitstressregs3',
 385                'jitstressregs4',
 386                'jitstressregs8',
 387                'jitstressregs0x10',
 388                'jitstressregs0x80',
 389                'jitstressregs0x1000',
 390                'jitstress2_jitstressregs1',
 391                'jitstress2_jitstressregs2',
 392                'jitstress2_jitstressregs3',
 393                'jitstress2_jitstressregs4',
 394                'jitstress2_jitstressregs8',
 395                'jitstress2_jitstressregs0x10',
 396                'jitstress2_jitstressregs0x80',
 397                'jitstress2_jitstressregs0x1000',
 398                'tailcallstress',
 399                // 'jitsse2only'                          // Only relevant to xarch
 400                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 401                // 'jitincompletehwintrinsic'
 402                // 'jitx86hwintrinsicnoavx'
 403                // 'jitx86hwintrinsicnoavx2'
 404                // 'jitx86hwintrinsicnosimd'
 405                // 'jitnox86hwintrinsic'
 406                'corefx_baseline',
 407                'corefx_minopts',
 408                'corefx_tieredcompilation',
 409                'corefx_jitstress1',
 410                'corefx_jitstress2',
 411                'corefx_jitstressregs1',
 412                'corefx_jitstressregs2',
 413                'corefx_jitstressregs3',
 414                'corefx_jitstressregs4',
 415                'corefx_jitstressregs8',
 416                'corefx_jitstressregs0x10',
 417                'corefx_jitstressregs0x80',
 418                'corefx_jitstressregs0x1000',
 419                'gcstress0x3',
 420                'gcstress0xc',
 421                'zapdisable',
 422                'heapverify1',
 423                'gcstress0xc_zapdisable',
 424                'gcstress0xc_zapdisable_jitstress2',
 425                'gcstress0xc_zapdisable_heapverify1',
 426                'gcstress0xc_jitstress1',
 427                'gcstress0xc_jitstress2',
 428                'gcstress0xc_minopts_heapverify1'
 429     ]
 430
 431     def static validLinuxArm64Scenarios = [
 432                'innerloop',
 433                'normal',
 434                // 'ilrt'
 435                'r2r',
 436                // 'longgc'
 437                // 'formatting'
 438                // 'gcsimulator'
 439                // 'jitdiff'
 440                // 'standalone_gc'
 441                // 'gc_reliability_framework'
 442                // 'illink'
 443                'r2r_jitstress1',
 444                'r2r_jitstress2',
 445                'r2r_jitstress1_tiered',
 446                'r2r_jitstress2_tiered',
 447                'r2r_jitstressregs1',
 448                'r2r_jitstressregs2',
 449                'r2r_jitstressregs3',
 450                'r2r_jitstressregs4',
 451                'r2r_jitstressregs8',
 452                'r2r_jitstressregs0x10',
 453                'r2r_jitstressregs0x80',
 454                'r2r_jitstressregs0x1000',
 455                'r2r_jitminopts',
 456                'r2r_jitforcerelocs',
 457                'r2r_gcstress15',
 458                'minopts',
 459                'tieredcompilation',
 460                'forcerelocs',
 461                'jitstress1',
 462                'jitstress2',
 463                'jitstress1_tiered',
 464                'jitstress2_tiered',
 465                'jitstressregs1',
 466                'jitstressregs2',
 467                'jitstressregs3',
 468                'jitstressregs4',
 469                'jitstressregs8',
 470                'jitstressregs0x10',
 471                'jitstressregs0x80',
 472                'jitstressregs0x1000',
 473                'jitstress2_jitstressregs1',
 474                'jitstress2_jitstressregs2',
 475                'jitstress2_jitstressregs3',
 476                'jitstress2_jitstressregs4',
 477                'jitstress2_jitstressregs8',
 478                'jitstress2_jitstressregs0x10',
 479                'jitstress2_jitstressregs0x80',
 480                'jitstress2_jitstressregs0x1000',
 481                'tailcallstress',
 482                // 'jitsse2only'                         // Only relevant to xarch
 483                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 484                // 'jitincompletehwintrinsic'
 485                // 'jitx86hwintrinsicnoavx'
 486                // 'jitx86hwintrinsicnoavx2'
 487                // 'jitx86hwintrinsicnosimd'
 488                // 'jitnox86hwintrinsic'
 489                'corefx_baseline',
 490                'corefx_minopts',
 491                'corefx_tieredcompilation',
 492                'corefx_jitstress1',
 493                'corefx_jitstress2',
 494                'corefx_jitstressregs1',
 495                'corefx_jitstressregs2',
 496                'corefx_jitstressregs3',
 497                'corefx_jitstressregs4',
 498                'corefx_jitstressregs8',
 499                'corefx_jitstressregs0x10',
 500                'corefx_jitstressregs0x80',
 501                'corefx_jitstressregs0x1000',
 502                'gcstress0x3',
 503                'gcstress0xc',
 504                'zapdisable',
 505                'heapverify1',
 506                'gcstress0xc_zapdisable',
 507                'gcstress0xc_zapdisable_jitstress2',
 508                'gcstress0xc_zapdisable_heapverify1',
 509                'gcstress0xc_jitstress1',
 510                'gcstress0xc_jitstress2',
 511                'gcstress0xc_minopts_heapverify1'
 512     ]
 513
 514     def static configurationList = ['Debug', 'Checked', 'Release']
 515
 516     // This is the set of architectures
 517     // Some of these are pseudo-architectures:
 518     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 519     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 520     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 521     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 522
 523     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 524     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 525 }
 526
 527 // **************************************************************
 528 // Create some specific views
 529 //
 530 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 531 // views based on a single regular expression. These views will be generated by adding a
 532 // specific set of jobs to them.
 533 //
 534 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 535 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 536 // those are required, don't add them (which simplifies the view pages, as well).
 537 // **************************************************************
 538
 539 class Views {
 540     def static MergeJobView = null
 541     def static PeriodicJobView = null
 542     def static ArchitectureViews = [:]
 543     def static OSViews = [:]
 544 }
 545
 546 // MergeJobView: include all jobs that execute when a PR change is merged.
 547 Views.MergeJobView = listView('Merge') {
 548     recurse()
 549     columns {
 550         status()
 551         weather()
 552         name()
 553         lastSuccess()
 554         lastFailure()
 555         lastDuration()
 556         buildButton()
 557     }
 558 }
 559
 560 // PeriodicJobView: include all jobs that execute on a schedule
 561 Views.PeriodicJobView = listView('Periodic') {
 562     recurse()
 563     columns {
 564         status()
 565         weather()
 566         name()
 567         lastSuccess()
 568         lastFailure()
 569         lastDuration()
 570         buildButton()
 571     }
 572 }
 573
 574 // Create a view for non-PR jobs for each architecture.
 575 Constants.architectureList.each { architecture ->
 576     Views.ArchitectureViews[architecture] = listView(architecture) {
 577         recurse()
 578         columns {
 579             status()
 580             weather()
 581             name()
 582             lastSuccess()
 583             lastFailure()
 584             lastDuration()
 585             buildButton()
 586         }
 587     }
 588 }
 589
 590 // Create a view for non-PR jobs for each OS.
 591 Constants.osList.each { os ->
 592     // Don't create one for the special 'Windows_NT_BuildOnly'
 593     if (os == 'Windows_NT_BuildOnly') {
 594         return
 595     }
 596     Views.OSViews[os] = listView(os) {
 597         recurse()
 598         columns {
 599             status()
 600             weather()
 601             name()
 602             lastSuccess()
 603             lastFailure()
 604             lastDuration()
 605             buildButton()
 606         }
 607     }
 608 }
 609
 610 def static addToMergeView(def job) {
 611     Views.MergeJobView.with {
 612         jobs {
 613             name(job.name)
 614         }
 615     }
 616 }
 617
 618 def static addToPeriodicView(def job) {
 619     Views.PeriodicJobView.with {
 620         jobs {
 621             name(job.name)
 622         }
 623     }
 624 }
 625
 626 def static addToViews(def job, def isPR, def architecture, def os) {
 627     if (isPR) {
 628         // No views want PR jobs currently.
 629         return
 630     }
 631
 632     // Add to architecture view.
 633     Views.ArchitectureViews[architecture].with {
 634         jobs {
 635             name(job.name)
 636         }
 637     }
 638
 639     // Add to OS view.
 640     Views.OSViews[os].with {
 641         jobs {
 642             name(job.name)
 643         }
 644     }
 645 }
 646
 647 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 648     addToPeriodicView(job)
 649     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 650 }
 651
 652 def static addGithubPushTriggerHelper(def job) {
 653     addToMergeView(job)
 654     Utilities.addGithubPushTrigger(job)
 655 }
 656
 657
 658 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 659     assert os instanceof String
 660     assert architecture instanceof String
 661
 662     def armArches = ['arm', 'armem', 'arm64']
 663
 664     if (!(architecture in armArches)) {
 665         assert options == null
 666         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 667
 668         return
 669     }
 670
 671     // This is an arm(64) job.
 672     //
 673     // There are several options.
 674     //
 675     // Windows_NT
 676     //
 677     // Arm32 (Build) -> latest-arm64
 678     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 679     // Arm32 (Test)  -> arm64-windows_nt
 680     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 681     //
 682     // Arm64 (Build) -> latest-arm64
 683     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 684     // Arm64 (Test)  -> arm64-windows_nt
 685     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 686     //
 687     // Ubuntu
 688     //
 689     // Arm32 emulator (Build, Test) -> arm-cross-latest
 690     //       |-> os == "Tizen" && (architecture == "armem")
 691     //
 692     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 693     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 694     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 695     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 696     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 697     //       |-> os == "Ubuntu" && (architecture == "arm")
 698     //
 699     // Arm64 (Build) -> arm64-cross-latest
 700     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 701     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 702     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 703     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 704     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 705
 706     // This has to be a arm arch
 707     assert architecture in armArches
 708     if (os == "Windows_NT") {
 709         // arm32/arm64 Windows jobs share the same machines for now
 710         def isBuild = options['use_arm64_build_machine'] == true
 711
 712         if (isBuild == true) {
 713             // Current set of machines with private Windows arm64 toolset:
 714             // Utilities.setMachineAffinity(job, os, 'latest-arm64')
 715             //
 716             // New set of machines with public Windows arm64 toolset, coming from Helix:
 717             job.with {
 718                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 719             }
 720         } else {
 721             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 722         }
 723     } else {
 724         assert os != 'Windows_NT'
 725
 726         if (architecture == 'arm64') {
 727             assert os == 'Ubuntu'
 728             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 729             def isBuild = (options != null) && (options['is_build_job'] == true)
 730             if (isFlow || isBuild) {
 731                 // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 732                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 733                 // Arm64 hardware.
 734                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 735             } else {
 736                 // Arm64 Linux test machines
 737                 if ((options != null) && (options['large_pages'] == true)) {
 738                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 739                 } else {
 740                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 741                 }
 742             }
 743         }
 744         else if (architecture == 'armem') {
 745             // arm emulator (Tizen). Build and test on same machine,
 746             // using Docker.
 747             assert os == 'Tizen'
 748             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 749         }
 750         else {
 751             // arm Ubuntu on hardware.
 752             assert architecture == 'arm'
 753             assert os == 'Ubuntu'
 754             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 755             def isBuild = (options != null) && (options['is_build_job'] == true)
 756             if (isFlow || isBuild) {
 757                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 758                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 759                 // arm hardware.
 760                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 761             } else {
 762                 // arm Ubuntu test machine
 763                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 764                 // Utilities.setMachineAffinity. Just add the machine affinity
 765                 // manually. We specify the Helix queue name here.
 766                 job.with {
 767                     label('ubuntu.1404.arm32.open')
 768                 }
 769             }
 770         }
 771     }
 772 }
 773
 774 // setJobMachineAffinity: compute the machine affinity options for a job,
 775 // then set the job with those affinity options.
 776 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 777 {
 778     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 779            (!isBuildJob && isTestJob  && !isFlowJob) ||
 780            (!isBuildJob && !isTestJob && isFlowJob)
 781
 782     def affinityOptions = null
 783     def affinityArchitecture = architecture
 784
 785     if (os == "Windows_NT") {
 786         if (architecture in Constants.armWindowsCrossArchitectureList) {
 787             if (isBuildJob) {
 788                 affinityOptions = [ "use_arm64_build_machine" : true ]
 789             } else if (isTestJob) {
 790                 affinityOptions = [ "use_arm64_build_machine" : false ]
 791             } else if (isFlowJob) {
 792                 // For the flow jobs set the machine affinity as x64
 793                 affinityArchitecture = 'x64'
 794             }
 795         }
 796     }
 797     else {
 798         if (architecture == 'arm64') {
 799             if (isBuildJob) {
 800                 affinityOptions = ['is_build_job': true]
 801             } else if (isFlowJob) {
 802                 affinityOptions = ['is_flow_job': true]
 803             } else if (isTestJob) {
 804                 affinityOptions = [ "large_pages" : false ]
 805             }
 806         }
 807         else if (architecture == 'arm') {
 808             if (isBuildJob) {
 809                 affinityOptions = ['is_build_job': true]
 810             } else if (isFlowJob) {
 811                 affinityOptions = ['is_flow_job': true]
 812             }
 813         }
 814     }
 815
 816     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 817 }
 818
 819 def static isGCStressRelatedTesting(def scenario) {
 820     // The 'r2r_gcstress15' scenario is a basic scenario.
 821     // Detect it and make it a GCStress related.
 822     if (scenario == 'r2r_gcstress15')
 823     {
 824         return true;
 825     }
 826
 827     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 828     def scenarioName = scenario.toLowerCase()
 829     def isGCStressTesting = false
 830     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 831         if (k in gcStressTestEnvVars) {
 832             isGCStressTesting = true;
 833         }
 834     }
 835     return isGCStressTesting
 836 }
 837
 838 def static isCoreFxScenario(def scenario) {
 839     def corefx_prefix = 'corefx_'
 840     if (scenario.length() < corefx_prefix.length()) {
 841         return false
 842     }
 843     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 844 }
 845
 846 def static isR2RBaselineScenario(def scenario) {
 847     return (scenario == 'r2r')
 848 }
 849
 850 def static isR2RStressScenario(def scenario) {
 851     return Constants.r2rStressScenarios.containsKey(scenario)
 852 }
 853
 854 def static isR2RScenario(def scenario) {
 855     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 856 }
 857
 858 def static isJitStressScenario(def scenario) {
 859     return Constants.jitStressModeScenarios.containsKey(scenario)
 860 }
 861
 862 def static isLongGc(def scenario) {
 863     return (scenario == 'longgc' || scenario == 'gcsimulator')
 864 }
 865
 866 def static isJitDiff(def scenario) {
 867     return (scenario == 'jitdiff')
 868 }
 869
 870 def static isGcReliabilityFramework(def scenario) {
 871     return (scenario == 'gc_reliability_framework')
 872 }
 873
 874 def static isArmWindowsScenario(def scenario) {
 875     return Constants.validArmWindowsScenarios.containsKey(scenario)
 876 }
 877
 878 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 879     if (isBuildOnly == true) {
 880         os = 'Windows_NT_BuildOnly'
 881     }
 882
 883     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 884     if (validOsPrTriggerArchConfigs != null) {
 885         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 886         if (validOsPrTriggerConfigs != null) {
 887             if (configuration in validOsPrTriggerConfigs) {
 888                 return true
 889             }
 890         }
 891     }
 892
 893     return false
 894 }
 895
 896 // This means the job builds and runs the 'Pri0' test set. This does not mean the job is
 897 // scheduled with a default PR trigger despite the correlation being true at the moment.
 898 def static isPri0TestScenario(def scenario) {
 899     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 900 }
 901
 902 def static getFxBranch(def branch) {
 903     def fxBranch = branch
 904     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 905     // branch even though CoreFX doesn't have such a branch.
 906     if (branch == 'dev/unix_test_workflow') {
 907         fxBranch = 'master'
 908     }
 909     return fxBranch
 910 }
 911
 912 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 913     // 2 hours (120 minutes) is the default timeout
 914     def timeout = 120
 915
 916     if (!isPri0TestScenario(scenario)) {
 917         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 918         timeout = 240
 919     }
 920
 921     if (!isBuildOnly) {
 922         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 923         if (isGCStressRelatedTesting(scenario)) {
 924             timeout = 4320
 925         }
 926         else if (isCoreFxScenario(scenario)) {
 927             timeout = 360
 928         }
 929         else if (isJitStressScenario(scenario)) {
 930             timeout = 300
 931         }
 932         else if (isR2RBaselineScenario(scenario)) {
 933             timeout = 240
 934         }
 935         else if (isLongGc(scenario)) {
 936             timeout = 1440
 937         }
 938         else if (isJitDiff(scenario)) {
 939             timeout = 240
 940         }
 941         else if (isGcReliabilityFramework(scenario)) {
 942             timeout = 1440
 943         }
 944         else if (architecture == 'armem' || architecture == 'arm64') {
 945             timeout = 240
 946         }
 947
 948         if (architecture == 'arm') {
 949             // ARM32 machines are particularly slow.
 950             timeout += 120
 951         }
 952     }
 953
 954     if (configuration == 'Debug') {
 955         // Debug runs can be very slow. Add an hour.
 956         timeout += 60
 957     }
 958
 959     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 960         // AltJit runs compile all methods twice.
 961         timeout *= 2
 962     }
 963
 964     // If we've changed the timeout from the default, set it in the job.
 965
 966     if (timeout != 120) {
 967         Utilities.setJobTimeout(newJob, timeout)
 968     }
 969 }
 970
 971 def static getJobFolder(def scenario) {
 972     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 973         return 'jitstress'
 974     }
 975     if (scenario == 'illink') {
 976         return 'illink'
 977     }
 978     return ''
 979 }
 980
 981 def static getStressModeDisplayName(def scenario) {
 982     def displayStr = ''
 983     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 984         def prefixLength = 'COMPlus_'.length()
 985         if (k.length() >= prefixLength) {
 986             def modeName = k.substring(prefixLength, k.length())
 987             displayStr += ' ' + modeName + '=' + v
 988         }
 989     }
 990
 991     if (isCoreFxScenario(scenario)) {
 992         displayStr = ('CoreFx ' + displayStr).trim()
 993     }
 994
 995     return displayStr
 996 }
 997
 998 def static getR2RDisplayName(def scenario) {
 999     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1000     def displayStr = scenario
1001     def prefixLength = 'r2r_'.length()
1002     if (displayStr.length() >= prefixLength) {
1003         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1004     } else if (scenario == 'r2r') {
1005         displayStr = "R2R"
1006     }
1007     return displayStr
1008 }
1009
1010 def static getScenarioDisplayString(def scenario) {
1011     switch (scenario) {
1012         case 'innerloop':
1013             return "Innerloop Build and Test"
1014
1015         case 'no_tiered_compilation_innerloop':
1016             def displayStr = getStressModeDisplayName(scenario)
1017             return "Innerloop Build and Test (Jit - ${displayStr})"
1018
1019         case 'corefx_innerloop':
1020             return "CoreFX Tests"
1021
1022         case 'normal':
1023             return "Build and Test"
1024
1025         case 'jitdiff':
1026             return "Jit Diff Build and Test"
1027
1028         case 'ilrt':
1029             return "IL RoundTrip Build and Test"
1030
1031         case 'longgc':
1032             return "Long-Running GC Build & Test"
1033
1034         case 'gcsimulator':
1035             return "GC Simulator"
1036
1037         case 'standalone_gc':
1038             return "Standalone GC"
1039
1040         case 'gc_reliability_framework':
1041             return "GC Reliability Framework"
1042
1043         case 'illink':
1044             return "via ILLink"
1045
1046         default:
1047             if (isJitStressScenario(scenario)) {
1048                 def displayStr = getStressModeDisplayName(scenario)
1049                 return "Build and Test (Jit - ${displayStr})"
1050             }
1051             else if (isR2RScenario(scenario)) {
1052                 def displayStr = getR2RDisplayName(scenario)
1053                 return "${displayStr} Build and Test"
1054             }
1055             else {
1056                 return "${scenario}"
1057             }
1058             break
1059     }
1060
1061     println("Unknown scenario: ${scenario}");
1062     assert false
1063 }
1064
1065 //
1066 // Functions to create an environment script.
1067 //      envScriptCreate -- initialize the script (call first)
1068 //      envScriptFinalize -- finalize the script (call last)
1069 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1070 //      envScriptAppendExistingScript -- append an existing script to the generated script
1071 //
1072 // Each script returns a string of commands. Concatenate all the strings together before
1073 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1074 //
1075
1076 // Initialize the environment setting script.
1077 def static envScriptCreate(def os, def stepScriptLocation) {
1078     def stepScript = ''
1079     if (os == 'Windows_NT') {
1080         stepScript += "echo Creating TestEnv script\r\n"
1081         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1082
1083         // Create at least an empty script.
1084         stepScript += "echo. > ${stepScriptLocation}\r\n"
1085     }
1086     else {
1087         stepScript += "echo Creating environment setting script\n"
1088         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1089     }
1090
1091     return stepScript
1092 }
1093
1094 // Generates the string for setting stress mode variables.
1095 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1096     def stepScript = ''
1097     if (os == 'Windows_NT') {
1098         stressModeVars.each{ k, v ->
1099             // Write out what we are writing to the script file
1100             stepScript += "echo Setting ${k}=${v}\r\n"
1101             // Write out the set itself to the script file`
1102             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1103         }
1104     }
1105     else {
1106         stressModeVars.each{ k, v ->
1107             // Write out what we are writing to the script file
1108             stepScript += "echo Setting ${k}=${v}\n"
1109             // Write out the set itself to the script file`
1110             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1111         }
1112     }
1113
1114     return stepScript
1115 }
1116
1117 // Append an existing script to an environment script.
1118 // Returns string of commands to do this.
1119 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1120     assert (os == 'Windows_NT')
1121     def stepScript = ''
1122
1123     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1124     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1125
1126     return stepScript
1127 }
1128
1129 // Finalize an environment setting script.
1130 // Returns string of commands to do this.
1131 def static envScriptFinalize(def os, def stepScriptLocation) {
1132     def stepScript = ''
1133
1134     if (os == 'Windows_NT') {
1135         // Display the resulting script. This is useful when looking at the output log file.
1136         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1137         stepScript += "type ${stepScriptLocation}\r\n"
1138     }
1139     else {
1140         stepScript += "chmod +x ${stepScriptLocation}\n"
1141     }
1142
1143     return stepScript
1144 }
1145
1146 def static isNeedDocker(def architecture, def os, def isBuild) {
1147     if (isBuild) {
1148         if (architecture == 'x86' && os == 'Ubuntu') {
1149             return true
1150         }
1151         else if (architecture == 'armem') {
1152             return true
1153         }
1154         else if (architecture == 'arm') {
1155             if (os == 'Ubuntu') {
1156                 return true
1157             }
1158         }
1159         else if (architecture == 'arm64') {
1160             if (os == 'Ubuntu') {
1161                 return true
1162             }
1163         }
1164     }
1165     else {
1166         if (architecture == 'x86' && os == 'Ubuntu') {
1167             return true
1168         }
1169     }
1170     return false
1171 }
1172
1173 def static getDockerImageName(def architecture, def os, def isBuild) {
1174     // We must change some docker private images to official later
1175     if (isBuild) {
1176         if (architecture == 'x86' && os == 'Ubuntu') {
1177             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1178         }
1179         else if (architecture == 'armem') {
1180             if (os == 'Tizen') {
1181                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1182             }
1183         }
1184         else if (architecture == 'arm') {
1185             if (os == 'Ubuntu') {
1186                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1187             }
1188         }
1189         else if (architecture == 'arm64') {
1190             if (os == 'Ubuntu') {
1191                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1192             }
1193         }
1194     }
1195     else {
1196         if (architecture == 'x86' && os == 'Ubuntu') {
1197             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1198         }
1199     }
1200     println("Unknown architecture to use docker: ${architecture} ${os}");
1201     assert false
1202 }
1203
1204
1205 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1206 // and only allowing using this hardware in some specific branches.
1207 def static jobRequiresLimitedHardware(def architecture, def os) {
1208     if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
1209         // These test jobs require ARM64 hardware
1210         return true
1211     }
1212     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1213         // These test jobs require Linux/arm32 hardware
1214         return true
1215     }
1216     else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
1217         // These test jobs require Linux/arm64 hardware
1218         return true
1219     }
1220     else {
1221         return false
1222     }
1223 }
1224
1225 // Calculates the name of the build job based on some typical parameters.
1226 //
1227 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1228     // If the architecture is x64, do not add that info into the build name.
1229     // Need to change around some systems and other builds to pick up the right builds
1230     // to do that.
1231
1232     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1233     if (isBuildOnly) {
1234         suffix += '_bld'
1235     }
1236     def baseName = ''
1237     switch (architecture) {
1238         case 'x64':
1239             if (scenario == 'normal') {
1240                 // For now we leave x64 off of the name for compatibility with other jobs
1241                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1242             }
1243             else if (scenario == 'formatting') {
1244                 // we don't care about the configuration for the formatting job. It runs all configs
1245                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1246             }
1247             else {
1248                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1249             }
1250             break
1251         case 'arm64':
1252             if (os.toLowerCase() == "windows_nt") {
1253                 // These are cross builds
1254                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1255             }
1256             else {
1257                 // Defaults to a small page size set of machines.
1258                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1259             }
1260             break
1261         case 'armem':
1262             // These are cross builds
1263             assert os == 'Tizen'
1264             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1265             break
1266         case 'arm':
1267             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1268             break
1269         case 'x86':
1270         case 'x86_arm_altjit':
1271         case 'x64_arm64_altjit':
1272             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1273             break
1274         default:
1275             println("Unknown architecture: ${architecture}");
1276             assert false
1277             break
1278     }
1279
1280     return baseName + suffix
1281 }
1282
1283 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1284     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1285
1286     // Limited hardware is restricted for non-PR triggers to certain branches.
1287     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1288         return
1289     }
1290
1291     // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
1292     if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1293          return
1294     }
1295
1296     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1297     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1298     if (architecture == 'x86' && os == 'Ubuntu') {
1299         return
1300     }
1301
1302     // Check scenario.
1303     switch (scenario) {
1304         case 'innerloop':
1305         case 'no_tiered_compilation_innerloop':
1306             break
1307         case 'normal':
1308             switch (architecture) {
1309                 case 'x64':
1310                 case 'x86':
1311                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1312                         addPeriodicTriggerHelper(job, '@daily')
1313                     }
1314                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1315                         addGithubPushTriggerHelper(job)
1316                     }
1317                     break
1318                 case 'arm64':
1319                     if (os == 'Windows_NT') {
1320                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1321                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1322                             // Do the builds daily.
1323                             addPeriodicTriggerHelper(job, '@daily')
1324                         }
1325                     }
1326                     else {
1327                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1328                         if (isFlowJob) {
1329                             addPeriodicTriggerHelper(job, '@daily')
1330                         }
1331                     }
1332                     break
1333                 case 'arm':
1334                     if (os == 'Windows_NT') {
1335                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1336                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1337                             // Do the builds daily.
1338                             addPeriodicTriggerHelper(job, '@daily')
1339                         }
1340                     }
1341                     else {
1342                         assert os == 'Ubuntu'
1343                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1344                         if (isFlowJob) {
1345                             // Currently no push triggers, with limited arm Linux hardware.
1346                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1347                             addPeriodicTriggerHelper(job, '@daily')
1348                         }
1349                     }
1350                     break
1351                 case 'armem':
1352                     addGithubPushTriggerHelper(job)
1353                     break
1354                 case 'x86_arm_altjit':
1355                 case 'x64_arm64_altjit':
1356                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1357                     if (configuration == 'Checked') {
1358                         addGithubPushTriggerHelper(job)
1359                     }
1360                     break
1361                 default:
1362                     println("Unknown architecture: ${architecture}");
1363                     assert false
1364                     break
1365             }
1366             break
1367         case 'r2r':
1368             assert !(os in bidailyCrossList)
1369             // r2r gets a push trigger for checked/release
1370             if (configuration == 'Checked' || configuration == 'Release') {
1371                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1372                 if (architecture == 'x64' && os != 'OSX10.12') {
1373                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1374                     if (isFlowJob || os == 'Windows_NT') {
1375                         addGithubPushTriggerHelper(job)
1376                     }
1377                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1378                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1379                     if (isFlowJob) {
1380                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1381                     }
1382                 }
1383                 // For x86, only add per-commit jobs for Windows
1384                 else if (architecture == 'x86') {
1385                     if (os == 'Windows_NT') {
1386                         addGithubPushTriggerHelper(job)
1387                     }
1388                 }
1389                 // arm64 r2r jobs should only run weekly.
1390                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1391                 else if (architecture == 'arm64') {
1392                     if (os == 'Windows_NT') {
1393                         if (isFlowJob) {
1394                             addPeriodicTriggerHelper(job, '@weekly')
1395                         }
1396                     }
1397                 }
1398                 // arm r2r jobs should only run weekly.
1399                 else if (architecture == 'arm') {
1400                     if (isFlowJob) {
1401                         addPeriodicTriggerHelper(job, '@weekly')
1402                     }
1403                 }
1404             }
1405             break
1406         case 'r2r_jitstress1':
1407         case 'r2r_jitstress2':
1408         case 'r2r_jitstress1_tiered':
1409         case 'r2r_jitstress2_tiered':
1410         case 'r2r_jitstressregs1':
1411         case 'r2r_jitstressregs2':
1412         case 'r2r_jitstressregs3':
1413         case 'r2r_jitstressregs4':
1414         case 'r2r_jitstressregs8':
1415         case 'r2r_jitstressregs0x10':
1416         case 'r2r_jitstressregs0x80':
1417         case 'r2r_jitstressregs0x1000':
1418         case 'r2r_jitminopts':
1419         case 'r2r_jitforcerelocs':
1420         case 'r2r_gcstress15':
1421         case 'r2r_no_tiered_compilation':
1422             assert !(os in bidailyCrossList)
1423
1424             // GCStress=C is currently not supported on OS X
1425             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1426                 break
1427             }
1428
1429             // GC Stress 15 r2r gets a push trigger for checked/release
1430             if (configuration == 'Checked' || configuration == 'Release') {
1431                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1432                 if (architecture == 'x64') {
1433                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1434                     if (isFlowJob || os == 'Windows_NT') {
1435                         // Add a weekly periodic trigger
1436                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1437                     }
1438                 }
1439                 // For x86, only add periodic jobs for Windows
1440                 else if (architecture == 'x86') {
1441                     if (os == 'Windows_NT') {
1442                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1443                     }
1444                 }
1445                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1446                 else if (architecture == 'arm64') {
1447                     if (os == 'Windows_NT') {
1448                         if (isFlowJob) {
1449                             addPeriodicTriggerHelper(job, '@weekly')
1450                         }
1451                     }
1452                 }
1453                 else if (architecture == 'arm') {
1454                     if (isFlowJob) {
1455                         addPeriodicTriggerHelper(job, '@weekly')
1456                     }
1457                 }
1458             }
1459             break
1460         case 'longgc':
1461             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1462             assert configuration == 'Release'
1463             assert architecture == 'x64'
1464             addPeriodicTriggerHelper(job, '@daily')
1465             // TODO: Add once external email sending is available again
1466             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1467             break
1468         case 'gcsimulator':
1469             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1470             assert configuration == 'Release'
1471             assert architecture == 'x64'
1472             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1473             // TODO: Add once external email sending is available again
1474             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1475             break
1476         case 'standalone_gc':
1477             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1478             assert (configuration == 'Release' || configuration == 'Checked')
1479             // TODO: Add once external email sending is available again
1480             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1481             addPeriodicTriggerHelper(job, '@daily')
1482             break
1483         case 'gc_reliability_framework':
1484             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1485             assert (configuration == 'Release' || configuration == 'Checked')
1486             // Only triggered by phrase.
1487             break
1488         case 'ilrt':
1489             assert !(os in bidailyCrossList)
1490             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1491             if (architecture == 'x64' && configuration == 'Release') {
1492                 // We don't expect to see a job generated except in these scenarios
1493                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1494                 if (isFlowJob || os == 'Windows_NT') {
1495                     addPeriodicTriggerHelper(job, '@daily')
1496                 }
1497             }
1498             break
1499         case 'jitdiff':
1500             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1501             assert configuration == 'Checked'
1502             assert (architecture == 'x64' || architecture == 'x86')
1503             addGithubPushTriggerHelper(job)
1504             break
1505         case 'formatting':
1506             assert (os == 'Windows_NT' || os == "Ubuntu")
1507             assert architecture == 'x64'
1508             addGithubPushTriggerHelper(job)
1509             break
1510         case 'jitstressregs1':
1511         case 'jitstressregs2':
1512         case 'jitstressregs3':
1513         case 'jitstressregs4':
1514         case 'jitstressregs8':
1515         case 'jitstressregs0x10':
1516         case 'jitstressregs0x80':
1517         case 'jitstressregs0x1000':
1518         case 'minopts':
1519         case 'tieredcompilation':
1520         case 'no_tiered_compilation':
1521         case 'forcerelocs':
1522         case 'jitstress1':
1523         case 'jitstress2':
1524         case 'jitstress1_tiered':
1525         case 'jitstress2_tiered':
1526         case 'jitstress2_jitstressregs1':
1527         case 'jitstress2_jitstressregs2':
1528         case 'jitstress2_jitstressregs3':
1529         case 'jitstress2_jitstressregs4':
1530         case 'jitstress2_jitstressregs8':
1531         case 'jitstress2_jitstressregs0x10':
1532         case 'jitstress2_jitstressregs0x80':
1533         case 'jitstress2_jitstressregs0x1000':
1534         case 'tailcallstress':
1535         case 'jitsse2only':
1536         case 'jitnosimd':
1537         case 'jitnox86hwintrinsic':
1538         case 'jitincompletehwintrinsic':
1539         case 'jitx86hwintrinsicnoavx':
1540         case 'jitx86hwintrinsicnoavx2':
1541         case 'jitx86hwintrinsicnosimd':
1542         case 'corefx_baseline':
1543         case 'corefx_minopts':
1544         case 'corefx_tieredcompilation':
1545         case 'corefx_jitstress1':
1546         case 'corefx_jitstress2':
1547         case 'corefx_jitstressregs1':
1548         case 'corefx_jitstressregs2':
1549         case 'corefx_jitstressregs3':
1550         case 'corefx_jitstressregs4':
1551         case 'corefx_jitstressregs8':
1552         case 'corefx_jitstressregs0x10':
1553         case 'corefx_jitstressregs0x80':
1554         case 'corefx_jitstressregs0x1000':
1555         case 'zapdisable':
1556             if (os == 'CentOS7.1') {
1557                 break
1558             }
1559             if (os in bidailyCrossList) {
1560                 break
1561             }
1562             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1563             // build for the flow job. We don't need cron jobs for these. Note that the
1564             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1565             // function very early, so only non-Windows gets here.
1566             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1567                 break
1568             }
1569             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1570                 break
1571             }
1572             assert (os == 'Windows_NT') || (os in Constants.crossList)
1573             if (jobRequiresLimitedHardware(architecture, os)) {
1574                 addPeriodicTriggerHelper(job, '@weekly')
1575             }
1576             else {
1577                 addPeriodicTriggerHelper(job, '@daily')
1578             }
1579             break
1580         case 'heapverify1':
1581         case 'gcstress0x3':
1582             if (os == 'CentOS7.1') {
1583                 break
1584             }
1585             if (os in bidailyCrossList) {
1586                 break
1587             }
1588             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1589                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1590                 break
1591             }
1592             assert (os == 'Windows_NT') || (os in Constants.crossList)
1593             addPeriodicTriggerHelper(job, '@weekly')
1594             break
1595         case 'gcstress0xc':
1596         case 'gcstress0xc_zapdisable':
1597         case 'gcstress0xc_zapdisable_jitstress2':
1598         case 'gcstress0xc_zapdisable_heapverify1':
1599         case 'gcstress0xc_jitstress1':
1600         case 'gcstress0xc_jitstress2':
1601         case 'gcstress0xc_minopts_heapverify1':
1602             if (os == 'CentOS7.1') {
1603                 break
1604             }
1605             if (os == 'OSX10.12') {
1606                 // GCStress=C is currently not supported on OS X
1607                 break
1608             }
1609             if (os in bidailyCrossList) {
1610                 break
1611             }
1612             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1613                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1614                 break
1615             }
1616             assert (os == 'Windows_NT') || (os in Constants.crossList)
1617             addPeriodicTriggerHelper(job, '@weekly')
1618             break
1619
1620         case 'illink':
1621             // Testing on other operating systems TBD
1622             assert (os == 'Windows_NT' || os == 'Ubuntu')
1623             if (architecture == 'x64' || architecture == 'x86') {
1624                 if (configuration == 'Checked') {
1625                     addPeriodicTriggerHelper(job, '@daily')
1626                 }
1627             }
1628             break
1629
1630         default:
1631             println("Unknown scenario: ${scenario}");
1632             assert false
1633             break
1634     }
1635     return
1636 }
1637
1638 // **************************
1639 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1640 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1641 // builds will do a couple extra steps.
1642 // **************************
1643
1644 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1645 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1646 // it's just used for internal builds
1647 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1648 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1649     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1650
1651     if (isWindowsBuildOnlyJob) {
1652         return
1653     }
1654
1655     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1656     // Non pull request builds.
1657     if (!isPR) {
1658         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1659         return
1660     }
1661
1662     def arm64Users = [
1663         'adityamandaleeka',
1664         'AndyAyersMS',
1665         'briansull',
1666         'BruceForstall',
1667         'CarolEidt',
1668         'davidwrighton',
1669         'echesakovMSFT',
1670         'erozenfeld',
1671         'janvorli',
1672         'jashook',
1673         'pgodeq',
1674         'RussKeldorph',
1675         'sandreenko',
1676         'sdmaclea',
1677         'swaroop-sridhar',
1678         'jkotas',
1679         'markwilkie',
1680         'weshaggard'
1681     ]
1682
1683     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1684     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1685
1686     def contextString = ""
1687     def triggerString = ""
1688     def needsTrigger = true
1689     def isDefaultTrigger = false
1690     def isArm64PrivateJob = false
1691     def scenarioString = ""
1692
1693     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1694     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1695     // some jobs where the Debug configuration just does builds, no tests.
1696     //
1697     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1698     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1699     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1700     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1701
1702     if (architecture == 'armem') {
1703         assert os == 'Tizen'
1704         architecture = 'armel'
1705     }
1706
1707     switch (architecture) {
1708         case 'x64_arm64_altjit':
1709         case 'x86_arm_altjit':
1710             // TODO: for consistency, add "Build and Test" at end.
1711             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1712             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1713             break
1714
1715         case 'armel':
1716         case 'arm':
1717         case 'arm64':
1718             contextString = "${os} ${architecture} Cross ${configuration}"
1719             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1720
1721             if (scenario == 'innerloop') {
1722                 contextString += " Innerloop"
1723                 triggerString += "\\W+Innerloop"
1724             }
1725             else {
1726                 contextString += " ${scenario}"
1727                 triggerString += "\\W+${scenario}"
1728             }
1729
1730             if (configuration == 'Debug') {
1731                 contextString += " Build"
1732                 triggerString += "\\W+Build"
1733             }
1734             else {
1735                 contextString += " Build and Test"
1736                 triggerString += "\\W+Build and Test"
1737             }
1738
1739             triggerString += ".*"
1740             break
1741
1742         default:
1743             scenarioString = getScenarioDisplayString(scenario)
1744             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1745             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1746
1747             switch (scenario) {
1748                 case 'normal':
1749                     triggerString += "\\W+Build and Test.*"
1750                     break
1751
1752                 case 'corefx_innerloop': // maintain this asymmetry
1753                     triggerString += "\\W+CoreFX Tests.*"
1754                     break
1755
1756                 default:
1757                     triggerString += "\\W+${scenario}.*"
1758                     break
1759             }
1760
1761             triggerString += ".*"
1762             break
1763     }
1764
1765     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1766     // non-flow jobs that are only used as part of flow jobs.
1767
1768     switch (architecture) {
1769         case 'x64': // editor brace matching: {
1770             if (scenario == 'formatting') {
1771                 assert configuration == 'Checked'
1772                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1773                     isDefaultTrigger = true
1774                     contextString = "${os} ${architecture} Formatting"
1775                 }
1776                 break
1777             }
1778
1779             switch (os) {
1780                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1781                 case 'Debian8.4':
1782                 case 'RHEL7.2':
1783                     if (scenario == 'innerloop') {
1784                         assert !isFlowJob
1785                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1786                         isDefaultTrigger = true
1787                         break
1788                     }
1789
1790                     // fall through
1791
1792                 case 'Fedora24':
1793                 case 'Ubuntu16.04':
1794                 case 'Ubuntu16.10':
1795                     assert !isFlowJob
1796                     assert scenario != 'innerloop'
1797                     contextString = "${os} ${architecture} ${configuration} Build"
1798                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1799                     break
1800
1801                 case 'Ubuntu':
1802                     if (scenario == 'illink') {
1803                         break
1804                     }
1805                     else if (scenario == 'corefx_innerloop') {
1806                         if (configuration == 'Checked') {
1807                             isDefaultTrigger = true
1808                         }
1809                         break
1810                     }
1811
1812                     // fall through
1813
1814                 case 'OSX10.12':
1815                     // Triggers on the non-flow jobs aren't necessary here
1816                     // Corefx testing uses non-flow jobs.
1817                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1818                         needsTrigger = false
1819                         break
1820                     }
1821                     switch (scenario) {
1822                         case 'innerloop':
1823                             isDefaultTrigger = true
1824                             break
1825
1826                         case 'no_tiered_compilation_innerloop':
1827                             if (os == 'Ubuntu') {
1828                                 isDefaultTrigger = true
1829                             }
1830                             break
1831
1832                         default:
1833                             break
1834                     }
1835                     break
1836
1837                 case 'CentOS7.1':
1838                     switch (scenario) {
1839                         case 'innerloop':
1840                             // CentOS uses checked for default PR tests while debug is build only
1841                             if (configuration == 'Debug') {
1842                                 isDefaultTrigger = true
1843                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1844                                 break
1845                             }
1846
1847                             // Make sure this is a flow job to get build and test.
1848                             if (!isFlowJob) {
1849                                 needsTrigger = false
1850                                 break
1851                             }
1852
1853                             if (configuration == 'Checked') {
1854                                 assert job.name.contains("flow")
1855                                 isDefaultTrigger = true
1856                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1857                             }
1858                             break
1859
1860                         case 'normal':
1861                             // Make sure this is a flow job to get build and test.
1862                             if (!isFlowJob) {
1863                                 needsTrigger = false
1864                                 break
1865                             }
1866                             break
1867
1868                         default:
1869                             break
1870                     }
1871                     break
1872
1873                 case 'Windows_NT':
1874                     switch (scenario) {
1875                         case 'innerloop':
1876                         case 'no_tiered_compilation_innerloop':
1877                             isDefaultTrigger = true
1878                             break
1879
1880                         case 'corefx_innerloop':
1881                             if (configuration == 'Checked' || configuration == 'Release') {
1882                                 isDefaultTrigger = true
1883                             }
1884                             break
1885
1886                         default:
1887                             break
1888                     }
1889                     break
1890
1891                 default:
1892                     println("Unknown os: ${os}");
1893                     assert false
1894                     break
1895
1896             } // switch (os)
1897
1898             break
1899         // editor brace matching: }
1900
1901         case 'armel': // editor brace matching: {
1902             job.with {
1903                 publishers {
1904                     azureVMAgentPostBuildAction {
1905                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1906                     }
1907                 }
1908             }
1909
1910             switch (os) {
1911                 case 'Tizen':
1912                     if (scenario == 'innerloop') {
1913                         if (configuration == 'Checked') {
1914                             isDefaultTrigger = true
1915                         }
1916                     }
1917                     break
1918             }
1919
1920             break
1921         // editor brace matching: }
1922
1923         case 'arm':
1924         case 'arm64': // editor brace matching: {
1925
1926             switch (os) {
1927                 case 'Ubuntu':
1928                     // TODO: make arm and arm64 Ubuntu more alike
1929
1930                     if (architecture == 'arm') {
1931                         // Triggers on the non-flow jobs aren't necessary
1932                         if (!isFlowJob) {
1933                             needsTrigger = false
1934                             break
1935                         }
1936
1937                         switch (scenario) {
1938                             case 'innerloop':
1939                             case 'no_tiered_compilation_innerloop':
1940                                 if (configuration == 'Checked') {
1941                                     isDefaultTrigger = true
1942                                 }
1943                                 break
1944                         }
1945                     }
1946                     else {
1947                         assert architecture == 'arm64'
1948
1949                         switch (scenario) {
1950                             case 'innerloop':
1951                                 if (configuration == 'Debug' && !isFlowJob) {
1952                                     isDefaultTrigger = true
1953                                 }
1954                                 break
1955                         }
1956                     }
1957                     break
1958
1959                 case 'Windows_NT':
1960                     assert isArmWindowsScenario(scenario)
1961
1962                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1963                     // the non-flow Build job. All others need a trigger on the flow job.
1964                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1965                     if (isFlowJob != needsFlowJobTrigger) {
1966                         needsTrigger = false
1967                         break
1968                     }
1969
1970                     switch (scenario) {
1971                         case 'innerloop':
1972                             if (configuration == 'Debug') {
1973                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1974                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1975                                 // People in arm64Users will get both this and the Checked Build and Test job.
1976                                 isDefaultTrigger = true
1977                             } else if (configuration == 'Checked') {
1978                                 isDefaultTrigger = true
1979                                 isArm64PrivateJob = true
1980                             }
1981                             break
1982                         default:
1983                             isArm64PrivateJob = true
1984                             break
1985                     }
1986                     break
1987                 default:
1988                     println("NYI os: ${os}");
1989                     assert false
1990                     break
1991             }
1992             break
1993
1994         // editor brace matching: }
1995         case 'x86': // editor brace matching: {
1996             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1997             if (os == 'Ubuntu') {
1998                 // Triggers on the non-flow jobs aren't necessary here
1999                 if (!isFlowJob) {
2000                     needsTrigger = false
2001                     break
2002                 }
2003
2004                 // on-demand only for ubuntu x86
2005                 contextString = "${os} ${architecture} ${configuration} Build"
2006                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2007                 break
2008             }
2009             switch (scenario) {
2010                 case 'innerloop':
2011                 case 'no_tiered_compilation_innerloop':
2012                     isDefaultTrigger = true
2013                     break
2014                 default:
2015                     break
2016             }
2017             break
2018
2019         // editor brace matching: }
2020         case 'x64_arm64_altjit':
2021         case 'x86_arm_altjit':
2022             // Everything default
2023             break
2024
2025         default:
2026             println("Unknown architecture: ${architecture}");
2027             assert false
2028             break
2029     }
2030
2031     if (needsTrigger) {
2032         if (isArm64PrivateJob) {
2033             if (isDefaultTrigger) {
2034                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2035             }
2036             else {
2037                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2038             }
2039         }
2040         else {
2041             if (isDefaultTrigger) {
2042                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2043             }
2044             else {
2045                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2046             }
2047         }
2048     }
2049 }
2050
2051 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2052     def buildCommands = []
2053     def osGroup = getOSGroup(os)
2054     def lowerConfiguration = configuration.toLowerCase()
2055
2056     def priority = '1'
2057     if (isPri0TestScenario(scenario)) {
2058         priority = '0'
2059     }
2060
2061     def doCoreFxTesting = isCoreFxScenario(scenario)
2062
2063     // Calculate the build steps, archival, and xunit results
2064     switch (os) {
2065         case 'Windows_NT': // editor brace matching: {
2066             switch (architecture) {
2067                 case 'x64':
2068                 case 'x86':
2069                 case 'x86_arm_altjit':
2070                 case 'x64_arm64_altjit':
2071                     def arch = architecture
2072                     def buildOpts = ''
2073                     if (architecture == 'x86_arm_altjit') {
2074                         arch = 'x86'
2075                     }
2076                     else if (architecture == 'x64_arm64_altjit') {
2077                         arch = 'x64'
2078                     }
2079
2080                     if (scenario == 'formatting') {
2081                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2082                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2083                         break
2084                     }
2085
2086                     if (scenario == 'illink') {
2087                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2088                     }
2089
2090                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2091                     if ((lowerConfiguration == 'release') &&
2092                         (scenario in Constants.basicScenarios) &&
2093                         (architecture != 'x86_arm_altjit') &&
2094                         (architecture != 'x64_arm64_altjit')) {
2095
2096                         buildOpts += ' -enforcepgo'
2097                     }
2098
2099                     if (doCoreFxTesting) {
2100                         buildOpts += ' skiptests';
2101                     } else {
2102                         buildOpts += " -priority=${priority}"
2103                     }
2104
2105                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2106                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2107                     // 35 characters long.
2108
2109                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2110
2111                     if (!isBuildOnly) {
2112                         def runtestArguments = ''
2113                         def testOpts = 'collectdumps'
2114
2115                         if (isR2RScenario(scenario)) {
2116
2117                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2118                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2119                             // to cause the tests to be crossgen'ed.
2120
2121                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2122                                 testOpts += ' crossgenaltjit protononjit.dll'
2123                             } else {
2124                                 testOpts += ' crossgen'
2125                             }
2126
2127                             testOpts += ' runcrossgentests'
2128                         }
2129                         else if (scenario == 'jitdiff') {
2130                             testOpts += ' jitdisasm crossgen'
2131                         }
2132                         else if (scenario == 'ilrt') {
2133                             testOpts += ' ilasmroundtrip'
2134                         }
2135                         else if (isLongGc(scenario)) {
2136                             testOpts += " ${scenario} sequential"
2137                         }
2138                         else if (scenario == 'standalone_gc') {
2139                             testOpts += ' gcname clrgc.dll'
2140                         }
2141                         else if (scenario == 'illink') {
2142                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2143                         }
2144
2145                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2146                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2147                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2148                         // which is read by the xunit harness.)
2149                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2150                         {
2151                             def timeout = 1800000
2152                             testOpts += " timeout ${timeout}"
2153                         }
2154
2155                         // If we are running a stress mode, we should write out the set of key
2156                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2157
2158                         def envScriptPath = ''
2159                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2160                             def buildCommandsStr = ''
2161                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2162                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2163
2164                             if (isJitStressScenario(scenario)) {
2165                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2166                             }
2167                             else if (isR2RStressScenario(scenario)) {
2168                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2169                             }
2170
2171                             if (architecture == 'x86_arm_altjit') {
2172                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2173                             }
2174                             else if (architecture == 'x64_arm64_altjit') {
2175                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2176                             }
2177
2178                             envScriptFinalize(os, envScriptPath)
2179
2180                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2181                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2182                             // into a single script.
2183                             buildCommands += buildCommandsStr
2184                         }
2185                         else if (architecture == 'x86_arm_altjit') {
2186                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2187                         }
2188                         else if (architecture == 'x64_arm64_altjit') {
2189                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2190                         }
2191                         if (envScriptPath != '') {
2192                             testOpts += " TestEnv ${envScriptPath}"
2193                         }
2194
2195                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2196
2197                         if (doCoreFxTesting) {
2198                             if (scenario == 'corefx_innerloop') {
2199                                 // Create CORE_ROOT and testhost
2200                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2201                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2202
2203                                 // Archive and process (only) the test results
2204                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2205                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2206                             }
2207                             else {
2208                                 def workspaceRelativeFxRoot = "_/fx"
2209                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2210                                 def fxBranch = getFxBranch(branch)
2211
2212                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2213
2214                                 // Archive and process (only) the test results
2215                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2216                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2217
2218                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2219                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2220                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2221                             }
2222                         }
2223                         else if (isGcReliabilityFramework(scenario)) {
2224                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2225                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2226                         }
2227                         else {
2228                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2229                         }
2230                     } // end if (!isBuildOnly)
2231
2232                     if (!doCoreFxTesting) {
2233                         // Run the rest of the build
2234                         // Build the mscorlib for the other OS's
2235                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2236                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2237
2238                         if (arch == 'x64') {
2239                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2240                         }
2241
2242                         // Zip up the tests directory so that we don't use so much space/time copying
2243                         // 10s of thousands of files around.
2244                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2245
2246                         if (!isJitStressScenario(scenario)) {
2247                             // For Windows, pull full test results and test drops for x86/x64.
2248                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2249                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2250                         }
2251
2252                         if (scenario == 'jitdiff') {
2253                             // retrieve jit-dasm output for base commit, and run jit-diff
2254                             if (!isBuildOnly) {
2255                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2256                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2257                             }
2258                         }
2259
2260                         if (!isBuildOnly) {
2261                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2262                         }
2263                     }
2264                     break
2265                 case 'arm':
2266                 case 'arm64':
2267                     assert isArmWindowsScenario(scenario)
2268
2269                     def buildOpts = ''
2270
2271                     if (doCoreFxTesting) {
2272                         buildOpts += ' skiptests'
2273                     } else {
2274                         buildOpts += " -priority=${priority}"
2275                     }
2276
2277                     // This is now a build only job. Do not run tests. Use the flow job.
2278                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2279
2280                     if (doCoreFxTesting) {
2281                         assert isBuildOnly
2282
2283                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2284                         def envScriptPath = ''
2285                         def buildCommandsStr = ''
2286                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2287                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2288                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2289                         envScriptFinalize(os, envScriptPath)
2290                         buildCommands += buildCommandsStr
2291
2292                         def workspaceRelativeFxRootLinux = "_/fx"
2293                         def workspaceRelativeFxRootWin = "_\\fx"
2294                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2295                         def fxBranch = getFxBranch(branch)
2296
2297                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2298
2299                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2300                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2301                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2302
2303                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2304                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2305                     } else {
2306                         // Zip up the tests directory so that we don't use so much space/time copying
2307                         // 10s of thousands of files around.
2308                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2309
2310                         // Add archival.
2311                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2312                     }
2313                     break
2314                 default:
2315                     println("Unknown architecture: ${architecture}");
2316                     assert false
2317                     break
2318             }
2319             break
2320         // end case 'Windows_NT'; editor brace matching: }
2321         case 'Ubuntu':
2322         case 'Ubuntu16.04':
2323         case 'Ubuntu16.10':
2324         case 'Debian8.4':
2325         case 'OSX10.12':
2326         case 'CentOS7.1':
2327         case 'RHEL7.2':
2328         case 'Tizen':
2329         case 'Fedora24': // editor brace matching: {
2330             switch (architecture) {
2331                 case 'x64':
2332                 case 'x86':
2333                     if (architecture == 'x86' && os == 'Ubuntu') {
2334                         // build and PAL test
2335                         def dockerImage = getDockerImageName(architecture, os, true)
2336                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2337                         dockerImage = getDockerImageName(architecture, os, false)
2338                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2339                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2340                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2341                         break
2342                     }
2343
2344                     if (scenario == 'formatting') {
2345                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2346                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2347                         break
2348                     }
2349
2350                     if (scenario == 'illink') {
2351                         assert(os == 'Ubuntu')
2352                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2353                     }
2354
2355                     if (!doCoreFxTesting) {
2356                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2357                         // only on supported OS platforms.
2358                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2359                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2360
2361                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2362                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2363
2364                         // Basic archiving of the build
2365                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2366                         // And pal tests
2367                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2368                     }
2369                     else {
2370                         if (scenario == 'corefx_innerloop') {
2371                             assert os == 'Ubuntu' || 'OSX10.12'
2372                             assert architecture == 'x64'
2373
2374                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2375                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2376                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2377
2378                             break
2379                             // Archive and process (only) the test results
2380                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2381                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2382                         }
2383                         else {
2384                             // Corefx stress testing
2385                             assert os == 'Ubuntu'
2386                             assert architecture == 'x64'
2387                             assert lowerConfiguration == 'checked'
2388                             assert isJitStressScenario(scenario)
2389
2390                             // Build coreclr
2391                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2392
2393                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2394
2395                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2396                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2397                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2398                             buildCommands += envScriptCmds
2399
2400                             // Build and text corefx
2401                             def workspaceRelativeFxRoot = "_/fx"
2402                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2403                             def fxBranch = getFxBranch(branch)
2404
2405                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2406
2407                             // Archive and process (only) the test results
2408                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2409                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2410                         }
2411                     }
2412                     break
2413                 case 'armem':
2414                     // Emulator cross builds for ARM runs on Tizen currently
2415                     assert os == 'Tizen'
2416
2417                     def arm_abi = "armel"
2418                     def linuxCodeName = "tizen"
2419
2420                     // Unzip the Windows test binaries first. Exit with 0
2421                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2422
2423                     // Unpack the corefx binaries
2424                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2425                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2426
2427                     // Call the ARM CI script to cross build and test using docker
2428                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2429                     --mode=docker \\
2430                     --${arm_abi} \\
2431                     --linuxCodeName=${linuxCodeName} \\
2432                     --buildConfig=${lowerConfiguration} \\
2433                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2434                     --coreFxBinDir=./bin/CoreFxBinDir \\
2435                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2436
2437                     // Basic archiving of the build, no pal tests
2438                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2439                     break
2440                 case 'arm64':
2441                 case 'arm':
2442                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2443                     assert (os == 'Ubuntu')
2444
2445                     // Add some useful information to the log file. Ignore return codes.
2446                     buildCommands += "uname -a || true"
2447
2448                     def additionalOpts = ""
2449                     if (architecture == 'arm') {
2450                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2451                     }
2452
2453                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2454                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2455                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2456                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2457                     // ZIP up the generated CoreFX runtime and tests.
2458
2459                     def dockerImage = getDockerImageName(architecture, os, true)
2460                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2461
2462                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2463
2464                     if (doCoreFxTesting) {
2465                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2466
2467                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2468                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2469                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2470                         buildCommands += envScriptCmds
2471
2472                         // Build and text corefx
2473                         def workspaceRelativeFxRootLinux = "_/fx"
2474                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2475                         def fxBranch = getFxBranch(branch)
2476
2477                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2478
2479                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2480                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2481                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2482
2483                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2484                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2485                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2486                     }
2487                     else {
2488                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2489                         // download the appropriate CoreFX packages.
2490                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2491                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2492
2493                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2494
2495                         // ZIP up for the test job (created in the flow job code):
2496                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2497                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2498                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2499                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2500
2501                         // These commands are assumed to be run from the root of the workspace.
2502                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2503                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2504
2505                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2506                     }
2507
2508                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2509                     // the cleanup task in Jenkins can't remove.
2510                     newJob.with {
2511                         publishers {
2512                             azureVMAgentPostBuildAction {
2513                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2514                             }
2515                         }
2516                     }
2517                     break
2518                 default:
2519                     println("Unknown architecture: ${architecture}");
2520                     assert false
2521                     break
2522             }
2523             break
2524         // editor brace matching: }
2525         default:
2526             println("Unknown os: ${os}");
2527             assert false
2528             break
2529     } // os
2530
2531     return buildCommands
2532 }
2533
2534 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2535 // Returns true if the job should be generated.
2536 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2537 {
2538     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2539     if (scenario == 'innerloop' && !isPR) {
2540         return false
2541     }
2542
2543     // Run basic corefx tests only on PR-triggered jobs
2544     // Runs under Release and Checked
2545     if (scenario == 'corefx_innerloop' && !isPR) {
2546         return false
2547     }
2548
2549     // Tizen is only supported for armem architecture
2550     if (os == 'Tizen' && architecture != 'armem') {
2551         return false
2552     }
2553
2554     // Filter based on architecture.
2555
2556     switch (architecture) {
2557         case 'arm64':
2558         case 'arm':
2559             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2560                 return false
2561             }
2562             break
2563         case 'armem':
2564             if (os != 'Tizen') {
2565                 return false
2566             }
2567             break
2568         case 'x86_arm_altjit':
2569         case 'x64_arm64_altjit':
2570             if (os != 'Windows_NT') {
2571                 return false
2572             }
2573             break
2574         case 'x86':
2575             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2576                 return false
2577             }
2578             break
2579         case 'x64':
2580             // Everything implemented
2581             break
2582         default:
2583             println("Unknown architecture: ${architecture}")
2584             assert false
2585             break
2586     }
2587
2588     // Which (Windows) build only jobs are required?
2589
2590     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2591
2592     if (isBuildOnly) {
2593         switch (architecture) {
2594             case 'arm':
2595             case 'arm64':
2596                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2597                 if (!isCoreFxScenario(scenario)) {
2598                     return false
2599                 }
2600                 break
2601             case 'x64':
2602             case 'x86':
2603                 if (!isNormalOrInnerloop) {
2604                     return false
2605                 }
2606                 break
2607             default:
2608                 return false
2609         }
2610     }
2611
2612     // Filter based on scenario.
2613
2614     if (isJitStressScenario(scenario)) {
2615         if (configuration != 'Checked') {
2616             return false
2617         }
2618
2619         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
2620         if (!isEnabledOS) {
2621             return false
2622         }
2623
2624         switch (architecture) {
2625             case 'x64':
2626             case 'x86_arm_altjit':
2627             case 'x64_arm64_altjit':
2628                 break
2629
2630             case 'x86':
2631                 // x86 ubuntu: no stress modes
2632                 if (os == 'Ubuntu') {
2633                     return false
2634                 }
2635                 break
2636
2637             case 'arm':
2638                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2639                 // No "regular" Windows arm corefx jobs, e.g.
2640                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2641                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2642                 if (os == 'Windows_NT') {
2643                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2644                         return false
2645                     }
2646                 }
2647                 else {
2648                     if (!isCoreFxScenario(scenario)) {
2649                         return false
2650                     }
2651                 }
2652                 break
2653
2654             case 'arm64':
2655                 if (os == 'Windows_NT') {
2656                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2657                         return false
2658                     }
2659                 }
2660                 else {
2661                     if (!isCoreFxScenario(scenario)) {
2662                         return false
2663                     }
2664                 }
2665                 break
2666
2667             default:
2668                 // arm64: stress is handled through flow jobs.
2669                 // armem: no stress jobs for ARM emulator.
2670                 return false
2671         }
2672     }
2673     else if (isR2RScenario(scenario)) {
2674         if (os != 'Windows_NT') {
2675             return false
2676         }
2677
2678         if (isR2RBaselineScenario(scenario)) {
2679             // no need for Debug scenario; Checked is sufficient
2680             if (configuration != 'Checked' && configuration != 'Release') {
2681                 return false
2682             }
2683         }
2684         else if (isR2RStressScenario(scenario)) {
2685             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2686             if (configuration != 'Checked') {
2687                 return false
2688             }
2689         }
2690
2691         switch (architecture) {
2692             case 'arm':
2693             case 'arm64':
2694                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2695                 return false
2696
2697             default:
2698                 break
2699         }
2700     }
2701     else {
2702         // Skip scenarios
2703         switch (scenario) {
2704             case 'ilrt':
2705                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2706                 // the default scenario build
2707                 if (os != 'Windows_NT') {
2708                     return false
2709                 }
2710                 // Only x64 for now
2711                 if (architecture != 'x64') {
2712                     return false
2713                 }
2714                 // Release only
2715                 if (configuration != 'Release') {
2716                     return false
2717                 }
2718                 break
2719             case 'jitdiff':
2720                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2721                     return false
2722                 }
2723                 if (architecture != 'x64') {
2724                     return false
2725                 }
2726                 if (configuration != 'Checked') {
2727                     return false
2728                 }
2729                 break
2730             case 'longgc':
2731             case 'gcsimulator':
2732                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2733                     return false
2734                 }
2735                 if (architecture != 'x64') {
2736                     return false
2737                 }
2738                 if (configuration != 'Release') {
2739                     return false
2740                 }
2741                 break
2742             case 'gc_reliability_framework':
2743             case 'standalone_gc':
2744                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2745                     return false
2746                 }
2747
2748                 if (architecture != 'x64') {
2749                     return false
2750                 }
2751
2752                 if (configuration != 'Release' && configuration != 'Checked') {
2753                     return false
2754                 }
2755                 break
2756             // We only run Windows and Ubuntu x64 Checked for formatting right now
2757             case 'formatting':
2758                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2759                     return false
2760                 }
2761                 if (architecture != 'x64') {
2762                     return false
2763                 }
2764                 if (configuration != 'Checked') {
2765                     return false
2766                 }
2767                 break
2768             case 'illink':
2769                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2770                     return false
2771                 }
2772                 if (architecture != 'x64' && architecture != 'x86') {
2773                     return false
2774                 }
2775                 break
2776             case 'normal':
2777                 // Nothing skipped
2778                 break
2779             case 'innerloop':
2780                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2781                     return false
2782                 }
2783                 break
2784             case 'corefx_innerloop':
2785                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2786                     return false
2787                 }
2788                 if (architecture != 'x64') {
2789                     return false
2790                 }
2791                 break
2792             default:
2793                 println("Unknown scenario: ${scenario}")
2794                 assert false
2795                 break
2796         }
2797     }
2798
2799     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2800     // runtime behavior, not compile-time behavior, are not interesting.
2801     switch (architecture) {
2802         case 'x86_arm_altjit':
2803         case 'x64_arm64_altjit':
2804             if (isGCStressRelatedTesting(scenario)) {
2805                 return false
2806             }
2807             break
2808         default:
2809             break
2810     }
2811
2812     // The job was not filtered out, so we should generate it!
2813     return true
2814 }
2815
2816 Constants.allScenarios.each { scenario ->
2817     [true, false].each { isPR ->
2818         Constants.architectureList.each { architecture ->
2819             Constants.configurationList.each { configuration ->
2820                 Constants.osList.each { os ->
2821                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2822                     // and reset the os to Windows_NT
2823                     def isBuildOnly = false
2824                     if (os == 'Windows_NT_BuildOnly') {
2825                         isBuildOnly = true
2826                         os = 'Windows_NT'
2827                     }
2828
2829                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2830                         return
2831                     }
2832
2833                     // Calculate names
2834                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2835                     def folderName = getJobFolder(scenario)
2836
2837                     // Create the new job
2838                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2839
2840                     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
2841                     // "top-level" jobs. Build only jobs are such jobs.
2842                     if (!isBuildOnly)
2843                     {
2844                         addToViews(newJob, isPR, architecture, os)
2845                     }
2846
2847                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2848
2849                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2850                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2851                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2852
2853                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2854                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2855                     // artifacts produced.
2856                     if ((architecture == 'armem') && (os == 'Tizen')) {
2857                         // Define the Windows Tests and Corefx build job names
2858                         def lowerConfiguration = configuration.toLowerCase()
2859                         def WindowsTestsName = projectFolder + '/' +
2860                                                Utilities.getFullJobName(project,
2861                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2862                                                                         false)
2863                         def fxBranch = getFxBranch(branch)
2864                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2865                                            Utilities.getFolderName(fxBranch)
2866
2867                         def arm_abi = 'armel'
2868                         def corefx_os = 'tizen'
2869
2870                         // Let's use release CoreFX to test checked CoreCLR,
2871                         // because we do not generate checked CoreFX in CoreFX CI yet.
2872                         def corefx_lowerConfiguration = lowerConfiguration
2873                         if (lowerConfiguration == 'checked') {
2874                             corefx_lowerConfiguration = 'release'
2875                         }
2876
2877                         // Copy the Windows test binaries and the Corefx build binaries
2878                         newJob.with {
2879                             steps {
2880                                 copyArtifacts(WindowsTestsName) {
2881                                     includePatterns('bin/tests/tests.zip')
2882                                     buildSelector {
2883                                         latestSuccessful(true)
2884                                     }
2885                                 }
2886                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2887                                     includePatterns('bin/build.tar.gz')
2888                                     buildSelector {
2889                                         latestSuccessful(true)
2890                                     }
2891                                 }
2892                             } // steps
2893                         } // newJob.with
2894                     }
2895
2896                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2897
2898                     newJob.with {
2899                         steps {
2900                             if (os == 'Windows_NT') {
2901                                 buildCommands.each { buildCommand ->
2902                                     batchFile(buildCommand)
2903                                 }
2904                             }
2905                             else {
2906                                 buildCommands.each { buildCommand ->
2907                                     shell(buildCommand)
2908                                 }
2909                             }
2910                         } // steps
2911                     } // newJob.with
2912
2913                 } // os
2914             } // configuration
2915         } // architecture
2916     } // isPR
2917 } // scenario
2918
2919 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2920 // Returns the newly created job.
2921 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2922 {
2923     def osGroup = getOSGroup(os)
2924     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2925
2926     def jobFolder = getJobFolder(scenario)
2927     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2928         parameters {
2929             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2930         }
2931
2932         steps {
2933             // Set up the copies
2934
2935             // Coreclr build we are trying to test
2936             //
2937             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2938
2939             copyArtifacts(inputCoreCLRBuildName) {
2940                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2941                 buildSelector {
2942                     buildNumber('${CORECLR_BUILD}')
2943                 }
2944             }
2945
2946             if (isCoreFxScenario(scenario)) {
2947
2948                 // Only arm/arm64 supported for corefx testing now.
2949                 assert architecture == 'arm' || architecture == 'arm64'
2950
2951                 // Unzip CoreFx runtime
2952                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
2953
2954                 // Unzip CoreFx tests.
2955                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
2956
2957                 // Add the script to run the corefx tests
2958                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
2959                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2960                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
2961                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
2962
2963             } else { // !isCoreFxScenario(scenario)
2964
2965                 // Unzip tests.
2966                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
2967
2968                 def buildCommands = ""
2969
2970                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2971                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2972                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2973
2974                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2975                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2976
2977                 // For all jobs
2978                 addEnvVariable("CORE_ROOT", coreRootLocation)
2979                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2980                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2981
2982                 // If we are running a stress mode, we'll set those variables as well
2983                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2984                     def stressValues = null
2985                     if (isJitStressScenario(scenario)) {
2986                         stressValues = Constants.jitStressModeScenarios[scenario]
2987                     }
2988                     else {
2989                         stressValues = Constants.r2rStressScenarios[scenario]
2990                     }
2991
2992                     stressValues.each { key, value ->
2993                         addEnvVariable(key, value)
2994                     }
2995                 }
2996
2997                 if (isR2RScenario(scenario)) {
2998                     // Crossgen the framework assemblies.
2999                     buildCommands += """
3000 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3001 @goto skip_PrecompileAssembly
3002
3003 :PrecompileAssembly
3004 @REM Skip mscorlib since it is already precompiled.
3005 @if /I "%3" == "mscorlib.dll" exit /b 0
3006 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3007
3008 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3009 @if "%errorlevel%" == "-2146230517" (
3010     echo %2 is not a managed assembly.
3011 ) else if "%errorlevel%" == "-2146234344" (
3012     echo %2 is not a managed assembly.
3013 ) else if %errorlevel% neq 0 (
3014     echo Unable to precompile %2
3015 ) else (
3016     echo Precompiled %2
3017 )
3018 @exit /b 0
3019
3020 :skip_PrecompileAssembly
3021 """
3022
3023                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3024                     // crossgen on tests before running them.
3025                     addEnvVariable("RunCrossGen", "true")
3026                 } // isR2RScenario(scenario)
3027
3028                 // Create the smarty command
3029                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3030                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3031                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3032                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3033
3034                 // Exclude tests based on scenario.
3035                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3036                     addArchSpecificExclude(architecture, excludeTag)
3037                 }
3038
3039                 if (isPri0TestScenario(scenario)) {
3040                     addExclude("pri1")
3041                 }
3042
3043                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3044                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3045                 addExclude("LONG_RUNNING")
3046
3047                 smartyCommand += "/lstFile Tests.lst"
3048
3049                 def testListArch = [
3050                     'arm64': 'arm64',
3051                     'arm': 'arm'
3052                 ]
3053
3054                 def archLocation = testListArch[architecture]
3055
3056                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3057                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3058                 addCommand("${smartyCommand}")
3059
3060                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3061                 // However, we also need to remove all the variables that were set during this batch file, so we
3062                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3063                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3064                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3065                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3066                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3067                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3068
3069                 // ZIP up the smarty output, no matter what the smarty result.
3070                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3071
3072                 addCommand("echo %errorlevel%")
3073                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3074
3075                 // Use the smarty errorlevel as the script errorlevel.
3076                 addCommand("exit /b %__save_smarty_errorlevel%")
3077
3078                 batchFile(buildCommands)
3079             } // non-corefx testing
3080         } // steps
3081     } // job
3082
3083     if (!isCoreFxScenario(scenario)) {
3084         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3085
3086         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3087         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3088         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3089         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3090     }
3091
3092     return newJob
3093 }
3094
3095 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3096 // E.g., non-Windows tests.
3097 // Returns the newly created job.
3098 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3099 {
3100     def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
3101     def doCoreFxTesting = isCoreFxScenario(scenario)
3102
3103     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3104
3105     def osGroup = getOSGroup(os)
3106     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3107
3108     def testOpts = ''
3109     def useServerGC = false
3110
3111     // Enable Server GC for Ubuntu PR builds
3112     // REVIEW: why? Does this apply to all architectures? Why only PR?
3113     if (os == 'Ubuntu' && isPR) {
3114         testOpts += ' --useServerGC'
3115         useServerGC = true
3116     }
3117
3118     if (isR2RScenario(scenario)) {
3119
3120         testOpts += ' --crossgen --runcrossgentests'
3121
3122         if (scenario == 'r2r_jitstress1') {
3123             testOpts += ' --jitstress=1'
3124         }
3125         else if (scenario == 'r2r_jitstress2') {
3126             testOpts += ' --jitstress=2'
3127         }
3128         else if (scenario == 'r2r_jitstress1_tiered') {
3129             testOpts += ' --jitstress=1'
3130         }
3131         else if (scenario == 'r2r_jitstress2_tiered') {
3132             testOpts += ' --jitstress=2'
3133         }
3134         else if (scenario == 'r2r_jitstressregs1') {
3135             testOpts += ' --jitstressregs=1'
3136         }
3137         else if (scenario == 'r2r_jitstressregs2') {
3138             testOpts += ' --jitstressregs=2'
3139         }
3140         else if (scenario == 'r2r_jitstressregs3') {
3141             testOpts += ' --jitstressregs=3'
3142         }
3143         else if (scenario == 'r2r_jitstressregs4') {
3144             testOpts += ' --jitstressregs=4'
3145         }
3146         else if (scenario == 'r2r_jitstressregs8') {
3147             testOpts += ' --jitstressregs=8'
3148         }
3149         else if (scenario == 'r2r_jitstressregs0x10') {
3150             testOpts += ' --jitstressregs=0x10'
3151         }
3152         else if (scenario == 'r2r_jitstressregs0x80') {
3153             testOpts += ' --jitstressregs=0x80'
3154         }
3155         else if (scenario == 'r2r_jitstressregs0x1000') {
3156             testOpts += ' --jitstressregs=0x1000'
3157         }
3158         else if (scenario == 'r2r_jitminopts') {
3159             testOpts += ' --jitminopts'
3160         }
3161         else if (scenario == 'r2r_jitforcerelocs') {
3162             testOpts += ' --jitforcerelocs'
3163         }
3164         else if (scenario == 'r2r_gcstress15') {
3165             testOpts += ' --gcstresslevel=0xF'
3166         }
3167     }
3168     else if (scenario == 'jitdiff') {
3169         testOpts += ' --jitdisasm --crossgen'
3170     }
3171     else if (scenario == 'illink') {
3172         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3173     }
3174     else if (isLongGc(scenario)) {
3175         // Long GC tests behave very poorly when they are not
3176         // the only test running (many of them allocate until OOM).
3177         testOpts += ' --sequential'
3178
3179         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3180         // for running long GC and GCSimulator tests, respectively. We don't use them
3181         // here because using a playlist file produces much more readable output on the CI machines
3182         // and reduces running time.
3183         //
3184         // The Long GC playlist contains all of the tests that are
3185         // going to be run. The GCSimulator playlist contains all of
3186         // the GC simulator tests.
3187         if (scenario == 'longgc') {
3188             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3189         }
3190         else if (scenario == 'gcsimulator') {
3191             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3192         }
3193     }
3194     else if (isGcReliabilityFramework(scenario)) {
3195         testOpts += ' --build-overlay-only'
3196     }
3197     else if (scenario == 'standalone_gc') {
3198         if (osGroup == 'OSX') {
3199             testOpts += ' --gcname=libclrgc.dylib'
3200         }
3201         else if (osGroup == 'Linux') {
3202             testOpts += ' --gcname=libclrgc.so'
3203         }
3204         else {
3205             println("Unexpected OS group: ${osGroup} for os ${os}")
3206             assert false
3207         }
3208     }
3209
3210     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3211     // will be null in this case.
3212
3213     def jobFolder = getJobFolder(scenario)
3214     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3215         parameters {
3216             if (inputTestsBuildName != null) {
3217                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3218             }
3219             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3220         }
3221
3222         steps {
3223             // Set up the copies
3224
3225             // Coreclr build containing the tests and mscorlib
3226             // pri1 jobs still need to copy windows_nt built tests
3227             if (inputTestsBuildName != null) {
3228                 copyArtifacts(inputTestsBuildName) {
3229                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3230                     buildSelector {
3231                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3232                     }
3233                 }
3234             }
3235
3236             // Coreclr build we are trying to test
3237             //
3238             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3239
3240             copyArtifacts(inputCoreCLRBuildName) {
3241                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3242                 buildSelector {
3243                     buildNumber('${CORECLR_BUILD}')
3244                 }
3245             }
3246
3247             if (isUbuntuArmJob) {
3248                 // Add some useful information to the log file. Ignore return codes.
3249                 shell("uname -a || true")
3250             }
3251
3252             if (architecture == 'x86') {
3253                 shell("mkdir ./bin/CoreFxNative")
3254
3255                 def fxBranch = getFxBranch(branch)
3256                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3257
3258                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3259                     includePatterns('bin/build.tar.gz')
3260                     targetDirectory('bin/CoreFxNative')
3261                     buildSelector {
3262                         latestSuccessful(true)
3263                     }
3264                 }
3265
3266                 shell("mkdir ./bin/CoreFxBinDir")
3267                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3268             }
3269
3270             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3271             if (doCoreFxTesting) {
3272                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3273                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3274             }
3275             else {
3276                 // Unzip the tests first.  Exit with 0
3277                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3278                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3279             }
3280
3281             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3282             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3283             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3284             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3285             // copied correctly.
3286             if (!doCoreFxTesting) {
3287                 if (isUbuntuArmJob) {
3288                     def lowerConfiguration = configuration.toLowerCase()
3289                     shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3290                     shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3291                 }
3292                 else {
3293                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3294                 }
3295             }
3296
3297             // Execute the tests
3298             def runDocker = isNeedDocker(architecture, os, false)
3299             def dockerPrefix = ""
3300             def dockerCmd = ""
3301             if (runDocker) {
3302                 def dockerImage = getDockerImageName(architecture, os, false)
3303                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3304                 dockerCmd = dockerPrefix + "${dockerImage} "
3305             }
3306
3307             // If we are running a stress mode, we'll set those variables first.
3308             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3309             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3310                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3311                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3312                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3313                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3314                 shell("${envScriptCmds}")
3315                 testOpts += " --test-env=${scriptFileName}"
3316             }
3317
3318             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3319             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3320             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3321             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3322             if (isGCStressRelatedTesting(scenario)) {
3323                 if (architecture == 'x64') {
3324                     shell('./init-tools.sh')
3325                 }
3326             }
3327
3328             if (doCoreFxTesting) {
3329                 shell("""\
3330 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3331             }
3332             else {
3333                 def runScript = "${dockerCmd}./tests/runtest.sh"
3334
3335                 shell("""\
3336 ${runScript} \\
3337     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3338     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3339     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3340     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3341             }
3342
3343             if (isGcReliabilityFramework(scenario)) {
3344                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3345                 if (useServerGC) {
3346                     if (runDocker) {
3347                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3348                     }
3349                     else {
3350                         shell("export COMPlus_gcServer=1")
3351                     }
3352                 }
3353
3354                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3355             }
3356         } // steps
3357     } // job
3358
3359     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3360     if (os in ['Ubuntu']) {
3361         SummaryBuilder summaries = new SummaryBuilder()
3362         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3363         summaries.emit(newJob)
3364     }
3365
3366     if (doCoreFxTesting) {
3367         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3368         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3369     }
3370     else {
3371         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3372     }
3373
3374     return newJob
3375 }
3376
3377 // Create a test job that will be used by a flow job.
3378 // Returns the newly created job.
3379 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3380 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3381 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3382 {
3383     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3384
3385     def newJob = null
3386     if (windowsArmJob) {
3387         assert inputTestsBuildName == null
3388         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3389     } else {
3390         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3391     }
3392
3393     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3394
3395     if (scenario == 'jitdiff') {
3396         def osGroup = getOSGroup(os)
3397         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3398     }
3399
3400     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3401     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3402
3403     return newJob
3404 }
3405
3406 // Create a flow job to tie together a build job with the given test job.
3407 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3408 // Returns the new flow job.
3409 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3410 {
3411     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3412     // Linux CoreCLR test
3413     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3414     def jobFolder = getJobFolder(scenario)
3415
3416     def newFlowJob = null
3417
3418     if (inputTestsBuildName == null) {
3419         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3420                         buildFlow("""\
3421 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3422
3423 // And then build the test build
3424 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3425 """)
3426         }
3427         JobReport.Report.addReference(inputCoreCLRBuildName)
3428         JobReport.Report.addReference(fullTestJobName)
3429     }
3430     else {
3431         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3432                         buildFlow("""\
3433 // Build the input jobs in parallel
3434 parallel (
3435 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3436 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3437 )
3438
3439 // And then build the test build
3440 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3441                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3442 """)
3443         }
3444         JobReport.Report.addReference(inputCoreCLRBuildName)
3445         JobReport.Report.addReference(inputTestsBuildName)
3446         JobReport.Report.addReference(fullTestJobName)
3447     }
3448
3449     addToViews(newFlowJob, isPR, architecture, os)
3450
3451     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3452
3453     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3454     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3455
3456     return newFlowJob
3457 }
3458
3459 // Determine if we should generate a flow job for the given parameters.
3460 // Returns true if the job should be generated.
3461 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3462 {
3463     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3464     if (scenario == 'innerloop' && !isPR) {
3465         return false
3466     }
3467
3468     if (scenario == 'corefx_innerloop') {
3469         return false
3470     }
3471
3472     // Filter based on OS and architecture.
3473
3474     switch (architecture) {
3475         case 'arm64':
3476         case 'arm':
3477             if (os != "Ubuntu" && os != "Windows_NT") {
3478                 return false
3479             }
3480             break
3481         case 'x86':
3482             if (os != "Ubuntu") {
3483                 return false
3484             }
3485             break
3486         case 'x64':
3487             if (!(os in Constants.crossList)) {
3488                 return false
3489             }
3490             if (os == "Windows_NT") {
3491                 return false
3492             }
3493             break
3494         case 'armem':
3495         case 'x86_arm_altjit':
3496         case 'x64_arm64_altjit':
3497             // No flow jobs
3498             return false
3499         default:
3500             println("Unknown architecture: ${architecture}")
3501             assert false
3502             break
3503     }
3504
3505     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3506
3507     // Filter based on scenario in OS.
3508
3509     if (os == 'Windows_NT') {
3510         assert architecture == 'arm' || architecture == 'arm64'
3511         if (!isArmWindowsScenario(scenario)) {
3512             return false
3513         }
3514         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3515             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3516             // To do that, it doesn't require a flow job.
3517             return false
3518         }
3519     }
3520     else {
3521         // Non-Windows
3522         if (architecture == 'arm64') {
3523             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3524                 return false
3525             }
3526
3527             if (isNormalOrInnerloop && (configuration == 'Debug')) {
3528                 // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3529                 // To do that, it doesn't require a flow job.
3530                 return false
3531             }
3532         }
3533         else if (architecture == 'arm') {
3534             if (!(scenario in Constants.validLinuxArmScenarios)) {
3535                 return false
3536             }
3537         }
3538         else if (architecture == 'x86') {
3539             // Linux/x86 only want innerloop and default test
3540             if (!isNormalOrInnerloop) {
3541                 return false
3542             }
3543         }
3544         else if (architecture == 'x64') {
3545             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3546             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3547             // build from a Windows machine.
3548             if (isCoreFxScenario(scenario)) {
3549                 return false
3550             }
3551         }
3552     }
3553
3554     // For CentOS, we only want Checked/Release builds.
3555     if (os == 'CentOS7.1') {
3556         if (configuration != 'Checked' && configuration != 'Release') {
3557             return false
3558         }
3559         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3560             return false
3561         }
3562     }
3563
3564     // For RedHat and Debian, we only do Release builds.
3565     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3566         if (configuration != 'Release') {
3567             return false
3568         }
3569         if (!isNormalOrInnerloop) {
3570             return false
3571         }
3572     }
3573
3574     // Next, filter based on scenario.
3575
3576     if (isJitStressScenario(scenario)) {
3577         if (configuration != 'Checked') {
3578             return false
3579         }
3580     }
3581     else if (isR2RBaselineScenario(scenario)) {
3582         if (configuration != 'Checked' && configuration != 'Release') {
3583             return false
3584         }
3585     }
3586     else if (isR2RStressScenario(scenario)) {
3587         if (configuration != 'Checked') {
3588             return false
3589         }
3590     }
3591     else {
3592         // Skip scenarios
3593         switch (scenario) {
3594             case 'ilrt':
3595             case 'longgc':
3596             case 'gcsimulator':
3597                 // Long GC tests take a long time on non-Release builds
3598                 // ilrt is also Release only
3599                 if (configuration != 'Release') {
3600                     return false
3601                 }
3602                 break
3603
3604             case 'jitdiff':
3605                 if (configuration != 'Checked') {
3606                     return false
3607                 }
3608                 break
3609
3610             case 'gc_reliability_framework':
3611             case 'standalone_gc':
3612                 if (configuration != 'Release' && configuration != 'Checked') {
3613                     return false
3614                 }
3615                 break
3616
3617             case 'formatting':
3618                 return false
3619
3620             case 'illink':
3621                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3622                     return false
3623                 }
3624                 break
3625
3626             case 'normal':
3627                 // Nothing skipped
3628                 break
3629
3630             case 'innerloop':
3631                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3632                     return false
3633                 }
3634                 break
3635
3636             default:
3637                 println("Unknown scenario: ${scenario}")
3638                 assert false
3639                 break
3640         }
3641     }
3642
3643     // The job was not filtered out, so we should generate it!
3644     return true
3645 }
3646
3647 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3648 Constants.allScenarios.each { scenario ->
3649     [true, false].each { isPR ->
3650         Constants.architectureList.each { architecture ->
3651             Constants.configurationList.each { configuration ->
3652                 Constants.osList.each { os ->
3653
3654                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3655                         return
3656                     }
3657
3658                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3659                     def doCoreFxTesting = isCoreFxScenario(scenario)
3660
3661                     // Figure out the job name of the CoreCLR build the test will depend on.
3662
3663                     def inputCoreCLRBuildScenario = isPri0TestScenario(scenario) ? 'innerloop' : 'normal'
3664                     def inputCoreCLRBuildIsBuildOnly = false
3665                     if (doCoreFxTesting) {
3666                         // Every CoreFx test depends on its own unique build.
3667                         inputCoreCLRBuildScenario = scenario
3668                         if (windowsArmJob) {
3669                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3670                             inputCoreCLRBuildIsBuildOnly = true
3671                         }
3672                     }
3673                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3674                     def inputCoreCLRBuildName = projectFolder + '/' +
3675                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3676
3677                     // Figure out the name of the build job that the test job will depend on.
3678                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3679                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3680                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3681                     // CoreFX build creates the tests.
3682
3683                     def inputTestsBuildName = null
3684
3685                     if (!windowsArmJob && !doCoreFxTesting) {
3686                         def testBuildScenario = isPri0TestScenario(scenario) ? 'innerloop' : 'normal'
3687
3688                         def inputTestsBuildArch = architecture
3689                         if (architecture == "arm64") {
3690                             // Use the x64 test build for arm64 unix
3691                             inputTestsBuildArch = "x64"
3692                         }
3693                         else if (architecture == "arm") {
3694                             // Use the x86 test build for arm unix
3695                             inputTestsBuildArch = "x86"
3696                         }
3697
3698                         def inputTestsBuildIsBuildOnly = true
3699
3700                         inputTestsBuildName = projectFolder + '/' +
3701                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3702                     }
3703
3704                     // =============================================================================================
3705                     // Create the test job
3706                     // =============================================================================================
3707
3708                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3709
3710                     // =============================================================================================
3711                     // Create a build flow to join together the build and tests required to run this test.
3712                     // =============================================================================================
3713
3714                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3715                         // Do not create the flow job for RHEL jobs.
3716                         return
3717                     }
3718
3719                     def fullTestJobName = projectFolder + '/' + testJob.name
3720                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3721
3722                 } // os
3723             } // configuration
3724         } // architecture
3725     } // isPR
3726 } // scenario
3727
3728 JobReport.Report.generateJobReport(out)
3729
3730 // Make the call to generate the help job
3731 Utilities.createHelperJob(this, project, branch,
3732     "Welcome to the ${project} Repository",  // This is prepended to the help message
3733     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3734
3735 Utilities.addCROSSCheck(this, project, branch)