netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitecture(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return 'x86'
  39
  40         case 'arm64':
  41             return 'x64'
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  51     def static LimitedHardwareBranches = [
  52                'master']
  53
  54     // Innerloop build OS's
  55     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  56     // test execution in the build flow runs.  It generates the exact same build
  57     // as Windows_NT but without running the tests.
  58     def static osList = [
  59                'Ubuntu',
  60                'Debian8.4',
  61                'OSX10.12',
  62                'Windows_NT',
  63                'Windows_NT_BuildOnly',
  64                'CentOS7.1',
  65                'RHEL7.2',
  66                'Ubuntu16.04',
  67                'Ubuntu16.10',
  68                'Fedora24',
  69                'Tizen']
  70
  71     def static crossList = [
  72                'Ubuntu',
  73                'Debian8.4',
  74                'OSX10.12',
  75                'Windows_NT',
  76                'CentOS7.1',
  77                'RHEL7.2']
  78
  79     // This is a set of JIT stress modes combined with the set of variables that
  80     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  81     // the values are the environment variables
  82     def static jitStressModeScenarios = [
  83                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  84                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  85                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  86                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  87                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  88                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  89                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  90                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  91                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  92                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  93                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  94                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  95                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  96                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  97                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  98                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
  99                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 100                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 101                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 102                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 103                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 104                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 105                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 106                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 107                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 108                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 109                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 110                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 111                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 112                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 113                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 114                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 115                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 116                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 117                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 118                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 119                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 120                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 121                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 122                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 123                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 124                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 125                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 126                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 127                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 128                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 129                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 130                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 131                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 132                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 133                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 134                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 135                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 136                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 137                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 138                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 139     ]
 140
 141     // This is a set of ReadyToRun stress scenarios
 142     def static r2rStressScenarios = [
 143                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 144                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 145                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 146                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 147                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 148                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 149                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 150                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 151                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 152                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 153                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 154                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 155                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 156                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 157                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 158                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 159     ]
 160
 161     // This is the basic set of scenarios
 162     def static basicScenarios = [
 163                'innerloop',
 164                'normal',
 165                'ilrt',
 166                'r2r',
 167                'longgc',
 168                'formatting',
 169                'gcsimulator',
 170                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 171                'standalone_gc',
 172                'gc_reliability_framework',
 173                'illink',
 174                'corefx_innerloop',
 175                'crossgen_comparison']
 176
 177     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 178
 179     // Valid PR trigger combinations.
 180     def static prTriggeredValidInnerLoopCombos = [
 181         'Windows_NT': [
 182             'x64': [
 183                 'Checked'
 184             ],
 185             'x86': [
 186                 'Checked',
 187                 'Release'
 188             ],
 189             'arm': [
 190                 'Debug',
 191                 'Checked'
 192             ],
 193             'arm64': [
 194                 'Debug',
 195                 'Checked'
 196             ]
 197         ],
 198         'Windows_NT_BuildOnly': [
 199             'x64': [
 200                 'Checked',
 201                 'Release'
 202             ],
 203             'x86': [
 204                 'Checked',
 205                 'Release'
 206             ],
 207             'arm': [
 208                 'Checked'
 209             ],
 210         ],
 211         'Ubuntu': [
 212             'x64': [
 213                 'Checked'
 214             ],
 215             'arm64': [
 216                 'Debug'
 217             ],
 218             'arm': [
 219                 'Checked'
 220             ]
 221         ],
 222         'CentOS7.1': [
 223             'x64': [
 224                 'Debug',
 225                 'Checked'
 226             ]
 227         ],
 228         'OSX10.12': [
 229             'x64': [
 230                 'Checked'
 231             ]
 232         ],
 233         'Tizen': [
 234             'armem': [
 235                 'Checked'
 236             ]
 237         ]
 238     ]
 239
 240     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 241     // to Tests.lst file categories to exclude.
 242     //
 243     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 244     // and with the same values, with some commented out, for easier maintenance.
 245     //
 246     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 247     //
 248     def static validArmWindowsScenarios = [
 249                'innerloop':                              [],
 250                'normal':                                 [],
 251                // 'ilrt'
 252                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 253                // 'longgc'
 254                // 'formatting'
 255                // 'gcsimulator'
 256                // 'jitdiff'
 257                // 'standalone_gc'
 258                // 'gc_reliability_framework'
 259                // 'illink'
 260                // 'corefx_innerloop'
 261                // 'crossgen_comparison'
 262                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 275                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 277                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 278                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 279                'tieredcompilation':                      [],
 280                'no_tiered_compilation':                  [],
 281                'no_tiered_compilation_innerloop':        [],
 282                'forcerelocs':                            [],
 283                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 284                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 285                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 290                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 293                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 294                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 304                // 'jitsse2only'                          // Only relevant to xarch
 305                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 306                // 'jitincompletehwintrinsic'
 307                // 'jitx86hwintrinsicnoavx'
 308                // 'jitx86hwintrinsicnoavx2'
 309                // 'jitx86hwintrinsicnosimd'
 310                // 'jitnox86hwintrinsic'
 311                'corefx_baseline':                        [],    // corefx tests don't use smarty
 312                'corefx_minopts':                         [],    // corefx tests don't use smarty
 313                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 314                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 315                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 316                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 317                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 318                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 319                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 320                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 321                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 322                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 323                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 324                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 325                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 326                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 327                'heapverify1':                            [],
 328                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 329                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 330                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 331                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 332                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 333                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 334
 335                //
 336                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 337                //
 338
 339                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 340                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 341                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 342                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 343                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 344                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 345                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 346                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 347                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 348                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 349                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 350                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 351                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 352                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 353                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 354                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 355                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 356                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 357                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 358     ]
 359
 360     def static validLinuxArmScenarios = [
 361                'innerloop',
 362                'normal',
 363                // 'ilrt'
 364                'r2r',
 365                // 'longgc'
 366                // 'formatting'
 367                // 'gcsimulator'
 368                // 'jitdiff'
 369                // 'standalone_gc'
 370                // 'gc_reliability_framework'
 371                // 'illink'
 372                // 'corefx_innerloop'
 373                'crossgen_comparison',
 374                'r2r_jitstress1',
 375                'r2r_jitstress2',
 376                'r2r_jitstress1_tiered',
 377                'r2r_jitstress2_tiered',
 378                'r2r_jitstressregs1',
 379                'r2r_jitstressregs2',
 380                'r2r_jitstressregs3',
 381                'r2r_jitstressregs4',
 382                'r2r_jitstressregs8',
 383                'r2r_jitstressregs0x10',
 384                'r2r_jitstressregs0x80',
 385                'r2r_jitstressregs0x1000',
 386                'r2r_jitminopts',
 387                'r2r_jitforcerelocs',
 388                'r2r_gcstress15',
 389                'r2r_no_tiered_compilation',
 390                'minopts',
 391                'tieredcompilation',
 392                'no_tiered_compilation',
 393                'no_tiered_compilation_innerloop',
 394                'forcerelocs',
 395                'jitstress1',
 396                'jitstress2',
 397                'jitstress1_tiered',
 398                'jitstress2_tiered',
 399                'jitstressregs1',
 400                'jitstressregs2',
 401                'jitstressregs3',
 402                'jitstressregs4',
 403                'jitstressregs8',
 404                'jitstressregs0x10',
 405                'jitstressregs0x80',
 406                'jitstressregs0x1000',
 407                'jitstress2_jitstressregs1',
 408                'jitstress2_jitstressregs2',
 409                'jitstress2_jitstressregs3',
 410                'jitstress2_jitstressregs4',
 411                'jitstress2_jitstressregs8',
 412                'jitstress2_jitstressregs0x10',
 413                'jitstress2_jitstressregs0x80',
 414                'jitstress2_jitstressregs0x1000',
 415                'tailcallstress',
 416                // 'jitsse2only'                          // Only relevant to xarch
 417                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 418                // 'jitincompletehwintrinsic'
 419                // 'jitx86hwintrinsicnoavx'
 420                // 'jitx86hwintrinsicnoavx2'
 421                // 'jitx86hwintrinsicnosimd'
 422                // 'jitnox86hwintrinsic'
 423                'corefx_baseline',
 424                'corefx_minopts',
 425                'corefx_tieredcompilation',
 426                'corefx_jitstress1',
 427                'corefx_jitstress2',
 428                'corefx_jitstressregs1',
 429                'corefx_jitstressregs2',
 430                'corefx_jitstressregs3',
 431                'corefx_jitstressregs4',
 432                'corefx_jitstressregs8',
 433                'corefx_jitstressregs0x10',
 434                'corefx_jitstressregs0x80',
 435                'corefx_jitstressregs0x1000',
 436                'gcstress0x3',
 437                'gcstress0xc',
 438                'zapdisable',
 439                'heapverify1',
 440                'gcstress0xc_zapdisable',
 441                'gcstress0xc_zapdisable_jitstress2',
 442                'gcstress0xc_zapdisable_heapverify1',
 443                'gcstress0xc_jitstress1',
 444                'gcstress0xc_jitstress2',
 445                'gcstress0xc_minopts_heapverify1'
 446     ]
 447
 448     def static validLinuxArm64Scenarios = [
 449                'innerloop',
 450                'normal',
 451                // 'ilrt'
 452                'r2r',
 453                // 'longgc'
 454                // 'formatting'
 455                // 'gcsimulator'
 456                // 'jitdiff'
 457                // 'standalone_gc'
 458                // 'gc_reliability_framework'
 459                // 'illink'
 460                // 'corefx_innerloop'
 461                // 'crossgen_comparison'
 462                'r2r_jitstress1',
 463                'r2r_jitstress2',
 464                'r2r_jitstress1_tiered',
 465                'r2r_jitstress2_tiered',
 466                'r2r_jitstressregs1',
 467                'r2r_jitstressregs2',
 468                'r2r_jitstressregs3',
 469                'r2r_jitstressregs4',
 470                'r2r_jitstressregs8',
 471                'r2r_jitstressregs0x10',
 472                'r2r_jitstressregs0x80',
 473                'r2r_jitstressregs0x1000',
 474                'r2r_jitminopts',
 475                'r2r_jitforcerelocs',
 476                'r2r_gcstress15',
 477                'minopts',
 478                'tieredcompilation',
 479                'forcerelocs',
 480                'jitstress1',
 481                'jitstress2',
 482                'jitstress1_tiered',
 483                'jitstress2_tiered',
 484                'jitstressregs1',
 485                'jitstressregs2',
 486                'jitstressregs3',
 487                'jitstressregs4',
 488                'jitstressregs8',
 489                'jitstressregs0x10',
 490                'jitstressregs0x80',
 491                'jitstressregs0x1000',
 492                'jitstress2_jitstressregs1',
 493                'jitstress2_jitstressregs2',
 494                'jitstress2_jitstressregs3',
 495                'jitstress2_jitstressregs4',
 496                'jitstress2_jitstressregs8',
 497                'jitstress2_jitstressregs0x10',
 498                'jitstress2_jitstressregs0x80',
 499                'jitstress2_jitstressregs0x1000',
 500                'tailcallstress',
 501                // 'jitsse2only'                         // Only relevant to xarch
 502                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 503                // 'jitincompletehwintrinsic'
 504                // 'jitx86hwintrinsicnoavx'
 505                // 'jitx86hwintrinsicnoavx2'
 506                // 'jitx86hwintrinsicnosimd'
 507                // 'jitnox86hwintrinsic'
 508                'corefx_baseline',
 509                'corefx_minopts',
 510                'corefx_tieredcompilation',
 511                'corefx_jitstress1',
 512                'corefx_jitstress2',
 513                'corefx_jitstressregs1',
 514                'corefx_jitstressregs2',
 515                'corefx_jitstressregs3',
 516                'corefx_jitstressregs4',
 517                'corefx_jitstressregs8',
 518                'corefx_jitstressregs0x10',
 519                'corefx_jitstressregs0x80',
 520                'corefx_jitstressregs0x1000',
 521                'gcstress0x3',
 522                'gcstress0xc',
 523                'zapdisable',
 524                'heapverify1',
 525                'gcstress0xc_zapdisable',
 526                'gcstress0xc_zapdisable_jitstress2',
 527                'gcstress0xc_zapdisable_heapverify1',
 528                'gcstress0xc_jitstress1',
 529                'gcstress0xc_jitstress2',
 530                'gcstress0xc_minopts_heapverify1'
 531     ]
 532
 533     def static configurationList = ['Debug', 'Checked', 'Release']
 534
 535     // This is the set of architectures
 536     // Some of these are pseudo-architectures:
 537     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 538     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 539     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 540     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 541
 542     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 543     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 544 }
 545
 546 // **************************************************************
 547 // Create some specific views
 548 //
 549 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 550 // views based on a single regular expression. These views will be generated by adding a
 551 // specific set of jobs to them.
 552 //
 553 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 554 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 555 // those are required, don't add them (which simplifies the view pages, as well).
 556 // **************************************************************
 557
 558 class Views {
 559     def static MergeJobView = null
 560     def static PeriodicJobView = null
 561     def static ArchitectureViews = [:]
 562     def static OSViews = [:]
 563 }
 564
 565 // MergeJobView: include all jobs that execute when a PR change is merged.
 566 Views.MergeJobView = listView('Merge') {
 567     recurse()
 568     columns {
 569         status()
 570         weather()
 571         name()
 572         lastSuccess()
 573         lastFailure()
 574         lastDuration()
 575         buildButton()
 576     }
 577 }
 578
 579 // PeriodicJobView: include all jobs that execute on a schedule
 580 Views.PeriodicJobView = listView('Periodic') {
 581     recurse()
 582     columns {
 583         status()
 584         weather()
 585         name()
 586         lastSuccess()
 587         lastFailure()
 588         lastDuration()
 589         buildButton()
 590     }
 591 }
 592
 593 // Create a view for non-PR jobs for each architecture.
 594 Constants.architectureList.each { architecture ->
 595     Views.ArchitectureViews[architecture] = listView(architecture) {
 596         recurse()
 597         columns {
 598             status()
 599             weather()
 600             name()
 601             lastSuccess()
 602             lastFailure()
 603             lastDuration()
 604             buildButton()
 605         }
 606     }
 607 }
 608
 609 // Create a view for non-PR jobs for each OS.
 610 Constants.osList.each { os ->
 611     // Don't create one for the special 'Windows_NT_BuildOnly'
 612     if (os == 'Windows_NT_BuildOnly') {
 613         return
 614     }
 615     Views.OSViews[os] = listView(os) {
 616         recurse()
 617         columns {
 618             status()
 619             weather()
 620             name()
 621             lastSuccess()
 622             lastFailure()
 623             lastDuration()
 624             buildButton()
 625         }
 626     }
 627 }
 628
 629 def static addToMergeView(def job) {
 630     Views.MergeJobView.with {
 631         jobs {
 632             name(job.name)
 633         }
 634     }
 635 }
 636
 637 def static addToPeriodicView(def job) {
 638     Views.PeriodicJobView.with {
 639         jobs {
 640             name(job.name)
 641         }
 642     }
 643 }
 644
 645 def static addToViews(def job, def isPR, def architecture, def os) {
 646     if (isPR) {
 647         // No views want PR jobs currently.
 648         return
 649     }
 650
 651     // Add to architecture view.
 652     Views.ArchitectureViews[architecture].with {
 653         jobs {
 654             name(job.name)
 655         }
 656     }
 657
 658     // Add to OS view.
 659     Views.OSViews[os].with {
 660         jobs {
 661             name(job.name)
 662         }
 663     }
 664 }
 665
 666 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 667     addToPeriodicView(job)
 668     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 669 }
 670
 671 def static addGithubPushTriggerHelper(def job) {
 672     addToMergeView(job)
 673     Utilities.addGithubPushTrigger(job)
 674 }
 675
 676
 677 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 678     assert os instanceof String
 679     assert architecture instanceof String
 680
 681     def armArches = ['arm', 'armem', 'arm64']
 682
 683     if (!(architecture in armArches)) {
 684         assert options == null
 685         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 686
 687         return
 688     }
 689
 690     // This is an arm(64) job.
 691     //
 692     // There are several options.
 693     //
 694     // Windows_NT
 695     //
 696     // Arm32 (Build) -> latest-arm64
 697     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 698     // Arm32 (Test)  -> arm64-windows_nt
 699     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 700     //
 701     // Arm64 (Build) -> latest-arm64
 702     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 703     // Arm64 (Test)  -> arm64-windows_nt
 704     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 705     //
 706     // Ubuntu
 707     //
 708     // Arm32 emulator (Build, Test) -> arm-cross-latest
 709     //       |-> os == "Tizen" && (architecture == "armem")
 710     //
 711     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 712     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 713     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 714     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 715     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 716     //       |-> os == "Ubuntu" && (architecture == "arm")
 717     //
 718     // Arm64 (Build) -> arm64-cross-latest
 719     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 720     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 721     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 722     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 723     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 724
 725     // This has to be a arm arch
 726     assert architecture in armArches
 727     if (os == "Windows_NT") {
 728         // arm32/arm64 Windows jobs share the same machines for now
 729         def isBuild = options['use_arm64_build_machine'] == true
 730
 731         if (isBuild == true) {
 732             // Current set of machines with private Windows arm64 toolset:
 733             // Utilities.setMachineAffinity(job, os, 'latest-arm64')
 734             //
 735             // New set of machines with public Windows arm64 toolset, coming from Helix:
 736             job.with {
 737                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 738             }
 739         } else {
 740             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 741         }
 742     } else {
 743         assert os != 'Windows_NT'
 744
 745         if (architecture == 'arm64') {
 746             assert os == 'Ubuntu'
 747             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 748             def isBuild = (options != null) && (options['is_build_job'] == true)
 749             if (isFlow || isBuild) {
 750                 // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 751                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 752                 // Arm64 hardware.
 753                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 754             } else {
 755                 // Arm64 Linux test machines
 756                 if ((options != null) && (options['large_pages'] == true)) {
 757                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 758                 } else {
 759                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 760                 }
 761             }
 762         }
 763         else if (architecture == 'armem') {
 764             // arm emulator (Tizen). Build and test on same machine,
 765             // using Docker.
 766             assert os == 'Tizen'
 767             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 768         }
 769         else {
 770             // arm Ubuntu on hardware.
 771             assert architecture == 'arm'
 772             assert os == 'Ubuntu'
 773             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 774             def isBuild = (options != null) && (options['is_build_job'] == true)
 775             if (isFlow || isBuild) {
 776                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 777                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 778                 // arm hardware.
 779                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 780             } else {
 781                 // arm Ubuntu test machine
 782                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 783                 // Utilities.setMachineAffinity. Just add the machine affinity
 784                 // manually. We specify the Helix queue name here.
 785                 job.with {
 786                     label('ubuntu.1404.arm32.open')
 787                 }
 788             }
 789         }
 790     }
 791 }
 792
 793 // setJobMachineAffinity: compute the machine affinity options for a job,
 794 // then set the job with those affinity options.
 795 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 796 {
 797     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 798            (!isBuildJob && isTestJob  && !isFlowJob) ||
 799            (!isBuildJob && !isTestJob && isFlowJob)
 800
 801     def affinityOptions = null
 802     def affinityArchitecture = architecture
 803
 804     if (os == "Windows_NT") {
 805         if (architecture in Constants.armWindowsCrossArchitectureList) {
 806             if (isBuildJob) {
 807                 affinityOptions = [ "use_arm64_build_machine" : true ]
 808             } else if (isTestJob) {
 809                 affinityOptions = [ "use_arm64_build_machine" : false ]
 810             } else if (isFlowJob) {
 811                 // For the flow jobs set the machine affinity as x64
 812                 affinityArchitecture = 'x64'
 813             }
 814         }
 815     }
 816     else {
 817         if (architecture == 'arm64') {
 818             if (isBuildJob) {
 819                 affinityOptions = ['is_build_job': true]
 820             } else if (isFlowJob) {
 821                 affinityOptions = ['is_flow_job': true]
 822             } else if (isTestJob) {
 823                 affinityOptions = [ "large_pages" : false ]
 824             }
 825         }
 826         else if (architecture == 'arm') {
 827             if (isBuildJob) {
 828                 affinityOptions = ['is_build_job': true]
 829             } else if (isFlowJob) {
 830                 affinityOptions = ['is_flow_job': true]
 831             }
 832         }
 833     }
 834
 835     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 836 }
 837
 838 def static isGCStressRelatedTesting(def scenario) {
 839     // The 'r2r_gcstress15' scenario is a basic scenario.
 840     // Detect it and make it a GCStress related.
 841     if (scenario == 'r2r_gcstress15')
 842     {
 843         return true;
 844     }
 845
 846     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 847     def scenarioName = scenario.toLowerCase()
 848     def isGCStressTesting = false
 849     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 850         if (k in gcStressTestEnvVars) {
 851             isGCStressTesting = true;
 852         }
 853     }
 854     return isGCStressTesting
 855 }
 856
 857 def static isCoreFxScenario(def scenario) {
 858     def corefx_prefix = 'corefx_'
 859     if (scenario.length() < corefx_prefix.length()) {
 860         return false
 861     }
 862     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 863 }
 864
 865 def static isR2RBaselineScenario(def scenario) {
 866     return (scenario == 'r2r')
 867 }
 868
 869 def static isR2RStressScenario(def scenario) {
 870     return Constants.r2rStressScenarios.containsKey(scenario)
 871 }
 872
 873 def static isR2RScenario(def scenario) {
 874     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 875 }
 876
 877 def static isJitStressScenario(def scenario) {
 878     return Constants.jitStressModeScenarios.containsKey(scenario)
 879 }
 880
 881 def static isLongGc(def scenario) {
 882     return (scenario == 'longgc' || scenario == 'gcsimulator')
 883 }
 884
 885 def static isJitDiff(def scenario) {
 886     return (scenario == 'jitdiff')
 887 }
 888
 889 def static isGcReliabilityFramework(def scenario) {
 890     return (scenario == 'gc_reliability_framework')
 891 }
 892
 893 def static isArmWindowsScenario(def scenario) {
 894     return Constants.validArmWindowsScenarios.containsKey(scenario)
 895 }
 896
 897 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 898     if (isBuildOnly == true) {
 899         os = 'Windows_NT_BuildOnly'
 900     }
 901
 902     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 903     if (validOsPrTriggerArchConfigs != null) {
 904         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 905         if (validOsPrTriggerConfigs != null) {
 906             if (configuration in validOsPrTriggerConfigs) {
 907                 return true
 908             }
 909         }
 910     }
 911
 912     return false
 913 }
 914
 915 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 916 // scheduled with a default PR trigger despite the correlation being true at the moment.
 917 def static isInnerloopTestScenario(def scenario) {
 918     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 919 }
 920
 921 def static isCrossGenComparisonScenario(def scenario) {
 922     return (scenario == 'crossgen_comparison')
 923 }
 924
 925 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 926     assert isCrossGenComparisonScenario(scenario)
 927     return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
 928 }
 929
 930 def static getFxBranch(def branch) {
 931     def fxBranch = branch
 932     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 933     // branch even though CoreFX doesn't have such a branch.
 934     if (branch == 'dev/unix_test_workflow') {
 935         fxBranch = 'master'
 936     }
 937     return fxBranch
 938 }
 939
 940 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 941     // 2 hours (120 minutes) is the default timeout
 942     def timeout = 120
 943
 944     if (!isInnerloopTestScenario(scenario)) {
 945         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 946         timeout = 240
 947     }
 948
 949     if (!isBuildOnly) {
 950         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 951         if (isGCStressRelatedTesting(scenario)) {
 952             timeout = 4320
 953         }
 954         else if (isCoreFxScenario(scenario)) {
 955             timeout = 360
 956         }
 957         else if (isJitStressScenario(scenario)) {
 958             timeout = 300
 959         }
 960         else if (isR2RBaselineScenario(scenario)) {
 961             timeout = 240
 962         }
 963         else if (isLongGc(scenario)) {
 964             timeout = 1440
 965         }
 966         else if (isJitDiff(scenario)) {
 967             timeout = 240
 968         }
 969         else if (isGcReliabilityFramework(scenario)) {
 970             timeout = 1440
 971         }
 972         else if (architecture == 'armem' || architecture == 'arm64') {
 973             timeout = 240
 974         }
 975
 976         if (architecture == 'arm') {
 977             // ARM32 machines are particularly slow.
 978             timeout += 120
 979         }
 980     }
 981
 982     if (configuration == 'Debug') {
 983         // Debug runs can be very slow. Add an hour.
 984         timeout += 60
 985     }
 986
 987     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 988         // AltJit runs compile all methods twice.
 989         timeout *= 2
 990     }
 991
 992     // If we've changed the timeout from the default, set it in the job.
 993
 994     if (timeout != 120) {
 995         Utilities.setJobTimeout(newJob, timeout)
 996     }
 997 }
 998
 999 def static getJobFolder(def scenario) {
1000     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1001         return 'jitstress'
1002     }
1003     if (scenario == 'illink') {
1004         return 'illink'
1005     }
1006     return ''
1007 }
1008
1009 def static getStressModeDisplayName(def scenario) {
1010     def displayStr = ''
1011     Constants.jitStressModeScenarios[scenario].each{ k, v ->
1012         def prefixLength = 'COMPlus_'.length()
1013         if (k.length() >= prefixLength) {
1014             def modeName = k.substring(prefixLength, k.length())
1015             displayStr += ' ' + modeName + '=' + v
1016         }
1017     }
1018
1019     if (isCoreFxScenario(scenario)) {
1020         displayStr = ('CoreFx ' + displayStr).trim()
1021     }
1022
1023     return displayStr
1024 }
1025
1026 def static getR2RDisplayName(def scenario) {
1027     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1028     def displayStr = scenario
1029     def prefixLength = 'r2r_'.length()
1030     if (displayStr.length() >= prefixLength) {
1031         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1032     } else if (scenario == 'r2r') {
1033         displayStr = "R2R"
1034     }
1035     return displayStr
1036 }
1037
1038 def static getScenarioDisplayString(def scenario) {
1039     switch (scenario) {
1040         case 'innerloop':
1041             return "Innerloop Build and Test"
1042
1043         case 'no_tiered_compilation_innerloop':
1044             def displayStr = getStressModeDisplayName(scenario)
1045             return "Innerloop Build and Test (Jit - ${displayStr})"
1046
1047         case 'corefx_innerloop':
1048             return "CoreFX Tests"
1049
1050         case 'normal':
1051             return "Build and Test"
1052
1053         case 'jitdiff':
1054             return "Jit Diff Build and Test"
1055
1056         case 'ilrt':
1057             return "IL RoundTrip Build and Test"
1058
1059         case 'longgc':
1060             return "Long-Running GC Build & Test"
1061
1062         case 'gcsimulator':
1063             return "GC Simulator"
1064
1065         case 'standalone_gc':
1066             return "Standalone GC"
1067
1068         case 'gc_reliability_framework':
1069             return "GC Reliability Framework"
1070
1071         case 'illink':
1072             return "via ILLink"
1073
1074         default:
1075             if (isJitStressScenario(scenario)) {
1076                 def displayStr = getStressModeDisplayName(scenario)
1077                 return "Build and Test (Jit - ${displayStr})"
1078             }
1079             else if (isR2RScenario(scenario)) {
1080                 def displayStr = getR2RDisplayName(scenario)
1081                 return "${displayStr} Build and Test"
1082             }
1083             else {
1084                 return "${scenario}"
1085             }
1086             break
1087     }
1088
1089     println("Unknown scenario: ${scenario}");
1090     assert false
1091 }
1092
1093 //
1094 // Functions to create an environment script.
1095 //      envScriptCreate -- initialize the script (call first)
1096 //      envScriptFinalize -- finalize the script (call last)
1097 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1098 //      envScriptAppendExistingScript -- append an existing script to the generated script
1099 //
1100 // Each script returns a string of commands. Concatenate all the strings together before
1101 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1102 //
1103
1104 // Initialize the environment setting script.
1105 def static envScriptCreate(def os, def stepScriptLocation) {
1106     def stepScript = ''
1107     if (os == 'Windows_NT') {
1108         stepScript += "echo Creating TestEnv script\r\n"
1109         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1110
1111         // Create at least an empty script.
1112         stepScript += "echo. > ${stepScriptLocation}\r\n"
1113     }
1114     else {
1115         stepScript += "echo Creating environment setting script\n"
1116         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1117     }
1118
1119     return stepScript
1120 }
1121
1122 // Generates the string for setting stress mode variables.
1123 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1124     def stepScript = ''
1125     if (os == 'Windows_NT') {
1126         stressModeVars.each{ k, v ->
1127             // Write out what we are writing to the script file
1128             stepScript += "echo Setting ${k}=${v}\r\n"
1129             // Write out the set itself to the script file`
1130             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1131         }
1132     }
1133     else {
1134         stressModeVars.each{ k, v ->
1135             // Write out what we are writing to the script file
1136             stepScript += "echo Setting ${k}=${v}\n"
1137             // Write out the set itself to the script file`
1138             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1139         }
1140     }
1141
1142     return stepScript
1143 }
1144
1145 // Append an existing script to an environment script.
1146 // Returns string of commands to do this.
1147 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1148     assert (os == 'Windows_NT')
1149     def stepScript = ''
1150
1151     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1152     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1153
1154     return stepScript
1155 }
1156
1157 // Finalize an environment setting script.
1158 // Returns string of commands to do this.
1159 def static envScriptFinalize(def os, def stepScriptLocation) {
1160     def stepScript = ''
1161
1162     if (os == 'Windows_NT') {
1163         // Display the resulting script. This is useful when looking at the output log file.
1164         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1165         stepScript += "type ${stepScriptLocation}\r\n"
1166     }
1167     else {
1168         stepScript += "chmod +x ${stepScriptLocation}\n"
1169     }
1170
1171     return stepScript
1172 }
1173
1174 def static isNeedDocker(def architecture, def os, def isBuild) {
1175     if (isBuild) {
1176         if (architecture == 'x86' && os == 'Ubuntu') {
1177             return true
1178         }
1179         else if (architecture == 'armem') {
1180             return true
1181         }
1182         else if (architecture == 'arm') {
1183             if (os == 'Ubuntu') {
1184                 return true
1185             }
1186         }
1187         else if (architecture == 'arm64') {
1188             if (os == 'Ubuntu') {
1189                 return true
1190             }
1191         }
1192     }
1193     else {
1194         if (architecture == 'x86' && os == 'Ubuntu') {
1195             return true
1196         }
1197     }
1198     return false
1199 }
1200
1201 def static getDockerImageName(def architecture, def os, def isBuild) {
1202     // We must change some docker private images to official later
1203     if (isBuild) {
1204         if (architecture == 'x86' && os == 'Ubuntu') {
1205             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1206         }
1207         else if (architecture == 'armem') {
1208             if (os == 'Tizen') {
1209                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1210             }
1211         }
1212         else if (architecture == 'arm') {
1213             if (os == 'Ubuntu') {
1214                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1215             }
1216         }
1217         else if (architecture == 'arm64') {
1218             if (os == 'Ubuntu') {
1219                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1220             }
1221         }
1222     }
1223     else {
1224         if (architecture == 'x86' && os == 'Ubuntu') {
1225             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1226         }
1227     }
1228     println("Unknown architecture to use docker: ${architecture} ${os}");
1229     assert false
1230 }
1231
1232
1233 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1234 // and only allowing using this hardware in some specific branches.
1235 def static jobRequiresLimitedHardware(def architecture, def os) {
1236     if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
1237         // These test jobs require ARM64 hardware
1238         return true
1239     }
1240     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1241         // These test jobs require Linux/arm32 hardware
1242         return true
1243     }
1244     else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
1245         // These test jobs require Linux/arm64 hardware
1246         return true
1247     }
1248     else {
1249         return false
1250     }
1251 }
1252
1253 // Calculates the name of the build job based on some typical parameters.
1254 //
1255 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1256     // If the architecture is x64, do not add that info into the build name.
1257     // Need to change around some systems and other builds to pick up the right builds
1258     // to do that.
1259
1260     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1261     if (isBuildOnly) {
1262         suffix += '_bld'
1263     }
1264     def baseName = ''
1265     switch (architecture) {
1266         case 'x64':
1267             if (scenario == 'normal') {
1268                 // For now we leave x64 off of the name for compatibility with other jobs
1269                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1270             }
1271             else if (scenario == 'formatting') {
1272                 // we don't care about the configuration for the formatting job. It runs all configs
1273                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1274             }
1275             else {
1276                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1277             }
1278             break
1279         case 'arm64':
1280             if (os.toLowerCase() == "windows_nt") {
1281                 // These are cross builds
1282                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1283             }
1284             else {
1285                 // Defaults to a small page size set of machines.
1286                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1287             }
1288             break
1289         case 'armem':
1290             // These are cross builds
1291             assert os == 'Tizen'
1292             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1293             break
1294         case 'arm':
1295             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1296             break
1297         case 'x86':
1298         case 'x86_arm_altjit':
1299         case 'x64_arm64_altjit':
1300             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1301             break
1302         default:
1303             println("Unknown architecture: ${architecture}");
1304             assert false
1305             break
1306     }
1307
1308     return baseName + suffix
1309 }
1310
1311 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1312     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1313
1314     // Limited hardware is restricted for non-PR triggers to certain branches.
1315     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1316         return
1317     }
1318
1319     // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
1320     if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1321          return
1322     }
1323
1324     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1325     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1326     if (architecture == 'x86' && os == 'Ubuntu') {
1327         return
1328     }
1329
1330     // Check scenario.
1331     switch (scenario) {
1332         case 'innerloop':
1333         case 'no_tiered_compilation_innerloop':
1334         case 'crossgen_comparison':
1335             break
1336         case 'normal':
1337             switch (architecture) {
1338                 case 'x64':
1339                 case 'x86':
1340                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1341                         addPeriodicTriggerHelper(job, '@daily')
1342                     }
1343                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1344                         addGithubPushTriggerHelper(job)
1345                     }
1346                     break
1347                 case 'arm64':
1348                     if (os == 'Windows_NT') {
1349                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1350                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1351                             // Do the builds daily.
1352                             addPeriodicTriggerHelper(job, '@daily')
1353                         }
1354                     }
1355                     else {
1356                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1357                         if (isFlowJob) {
1358                             addPeriodicTriggerHelper(job, '@daily')
1359                         }
1360                     }
1361                     break
1362                 case 'arm':
1363                     if (os == 'Windows_NT') {
1364                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1365                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1366                             // Do the builds daily.
1367                             addPeriodicTriggerHelper(job, '@daily')
1368                         }
1369                     }
1370                     else {
1371                         assert os == 'Ubuntu'
1372                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1373                         if (isFlowJob) {
1374                             // Currently no push triggers, with limited arm Linux hardware.
1375                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1376                             addPeriodicTriggerHelper(job, '@daily')
1377                         }
1378                     }
1379                     break
1380                 case 'armem':
1381                     addGithubPushTriggerHelper(job)
1382                     break
1383                 case 'x86_arm_altjit':
1384                 case 'x64_arm64_altjit':
1385                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1386                     if (configuration == 'Checked') {
1387                         addGithubPushTriggerHelper(job)
1388                     }
1389                     break
1390                 default:
1391                     println("Unknown architecture: ${architecture}");
1392                     assert false
1393                     break
1394             }
1395             break
1396         case 'r2r':
1397             assert !(os in bidailyCrossList)
1398             // r2r gets a push trigger for checked/release
1399             if (configuration == 'Checked' || configuration == 'Release') {
1400                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1401                 if (architecture == 'x64' && os != 'OSX10.12') {
1402                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1403                     if (isFlowJob || os == 'Windows_NT') {
1404                         addGithubPushTriggerHelper(job)
1405                     }
1406                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1407                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1408                     if (isFlowJob) {
1409                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1410                     }
1411                 }
1412                 // For x86, only add per-commit jobs for Windows
1413                 else if (architecture == 'x86') {
1414                     if (os == 'Windows_NT') {
1415                         addGithubPushTriggerHelper(job)
1416                     }
1417                 }
1418                 // arm64 r2r jobs should only run weekly.
1419                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1420                 else if (architecture == 'arm64') {
1421                     if (os == 'Windows_NT') {
1422                         if (isFlowJob) {
1423                             addPeriodicTriggerHelper(job, '@weekly')
1424                         }
1425                     }
1426                 }
1427                 // arm r2r jobs should only run weekly.
1428                 else if (architecture == 'arm') {
1429                     if (isFlowJob) {
1430                         addPeriodicTriggerHelper(job, '@weekly')
1431                     }
1432                 }
1433             }
1434             break
1435         case 'r2r_jitstress1':
1436         case 'r2r_jitstress2':
1437         case 'r2r_jitstress1_tiered':
1438         case 'r2r_jitstress2_tiered':
1439         case 'r2r_jitstressregs1':
1440         case 'r2r_jitstressregs2':
1441         case 'r2r_jitstressregs3':
1442         case 'r2r_jitstressregs4':
1443         case 'r2r_jitstressregs8':
1444         case 'r2r_jitstressregs0x10':
1445         case 'r2r_jitstressregs0x80':
1446         case 'r2r_jitstressregs0x1000':
1447         case 'r2r_jitminopts':
1448         case 'r2r_jitforcerelocs':
1449         case 'r2r_gcstress15':
1450         case 'r2r_no_tiered_compilation':
1451             assert !(os in bidailyCrossList)
1452
1453             // GCStress=C is currently not supported on OS X
1454             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1455                 break
1456             }
1457
1458             // GC Stress 15 r2r gets a push trigger for checked/release
1459             if (configuration == 'Checked' || configuration == 'Release') {
1460                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1461                 if (architecture == 'x64') {
1462                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1463                     if (isFlowJob || os == 'Windows_NT') {
1464                         // Add a weekly periodic trigger
1465                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1466                     }
1467                 }
1468                 // For x86, only add periodic jobs for Windows
1469                 else if (architecture == 'x86') {
1470                     if (os == 'Windows_NT') {
1471                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1472                     }
1473                 }
1474                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1475                 else if (architecture == 'arm64') {
1476                     if (os == 'Windows_NT') {
1477                         if (isFlowJob) {
1478                             addPeriodicTriggerHelper(job, '@weekly')
1479                         }
1480                     }
1481                 }
1482                 else if (architecture == 'arm') {
1483                     if (isFlowJob) {
1484                         addPeriodicTriggerHelper(job, '@weekly')
1485                     }
1486                 }
1487             }
1488             break
1489         case 'longgc':
1490             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1491             assert configuration == 'Release'
1492             assert architecture == 'x64'
1493             addPeriodicTriggerHelper(job, '@daily')
1494             // TODO: Add once external email sending is available again
1495             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1496             break
1497         case 'gcsimulator':
1498             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1499             assert configuration == 'Release'
1500             assert architecture == 'x64'
1501             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1502             // TODO: Add once external email sending is available again
1503             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1504             break
1505         case 'standalone_gc':
1506             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1507             assert (configuration == 'Release' || configuration == 'Checked')
1508             // TODO: Add once external email sending is available again
1509             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1510             addPeriodicTriggerHelper(job, '@daily')
1511             break
1512         case 'gc_reliability_framework':
1513             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1514             assert (configuration == 'Release' || configuration == 'Checked')
1515             // Only triggered by phrase.
1516             break
1517         case 'ilrt':
1518             assert !(os in bidailyCrossList)
1519             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1520             if (architecture == 'x64' && configuration == 'Release') {
1521                 // We don't expect to see a job generated except in these scenarios
1522                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1523                 if (isFlowJob || os == 'Windows_NT') {
1524                     addPeriodicTriggerHelper(job, '@daily')
1525                 }
1526             }
1527             break
1528         case 'jitdiff':
1529             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1530             assert configuration == 'Checked'
1531             assert (architecture == 'x64' || architecture == 'x86')
1532             addGithubPushTriggerHelper(job)
1533             break
1534         case 'formatting':
1535             assert (os == 'Windows_NT' || os == "Ubuntu")
1536             assert architecture == 'x64'
1537             addGithubPushTriggerHelper(job)
1538             break
1539         case 'jitstressregs1':
1540         case 'jitstressregs2':
1541         case 'jitstressregs3':
1542         case 'jitstressregs4':
1543         case 'jitstressregs8':
1544         case 'jitstressregs0x10':
1545         case 'jitstressregs0x80':
1546         case 'jitstressregs0x1000':
1547         case 'minopts':
1548         case 'tieredcompilation':
1549         case 'no_tiered_compilation':
1550         case 'forcerelocs':
1551         case 'jitstress1':
1552         case 'jitstress2':
1553         case 'jitstress1_tiered':
1554         case 'jitstress2_tiered':
1555         case 'jitstress2_jitstressregs1':
1556         case 'jitstress2_jitstressregs2':
1557         case 'jitstress2_jitstressregs3':
1558         case 'jitstress2_jitstressregs4':
1559         case 'jitstress2_jitstressregs8':
1560         case 'jitstress2_jitstressregs0x10':
1561         case 'jitstress2_jitstressregs0x80':
1562         case 'jitstress2_jitstressregs0x1000':
1563         case 'tailcallstress':
1564         case 'jitsse2only':
1565         case 'jitnosimd':
1566         case 'jitnox86hwintrinsic':
1567         case 'jitincompletehwintrinsic':
1568         case 'jitx86hwintrinsicnoavx':
1569         case 'jitx86hwintrinsicnoavx2':
1570         case 'jitx86hwintrinsicnosimd':
1571         case 'corefx_baseline':
1572         case 'corefx_minopts':
1573         case 'corefx_tieredcompilation':
1574         case 'corefx_jitstress1':
1575         case 'corefx_jitstress2':
1576         case 'corefx_jitstressregs1':
1577         case 'corefx_jitstressregs2':
1578         case 'corefx_jitstressregs3':
1579         case 'corefx_jitstressregs4':
1580         case 'corefx_jitstressregs8':
1581         case 'corefx_jitstressregs0x10':
1582         case 'corefx_jitstressregs0x80':
1583         case 'corefx_jitstressregs0x1000':
1584         case 'zapdisable':
1585             if (os == 'CentOS7.1') {
1586                 break
1587             }
1588             if (os in bidailyCrossList) {
1589                 break
1590             }
1591             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1592             // build for the flow job. We don't need cron jobs for these. Note that the
1593             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1594             // function very early, so only non-Windows gets here.
1595             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1596                 break
1597             }
1598             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1599                 break
1600             }
1601             assert (os == 'Windows_NT') || (os in Constants.crossList)
1602             if (jobRequiresLimitedHardware(architecture, os)) {
1603                 addPeriodicTriggerHelper(job, '@weekly')
1604             }
1605             else {
1606                 addPeriodicTriggerHelper(job, '@daily')
1607             }
1608             break
1609         case 'heapverify1':
1610         case 'gcstress0x3':
1611             if (os == 'CentOS7.1') {
1612                 break
1613             }
1614             if (os in bidailyCrossList) {
1615                 break
1616             }
1617             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1618                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1619                 break
1620             }
1621             assert (os == 'Windows_NT') || (os in Constants.crossList)
1622             addPeriodicTriggerHelper(job, '@weekly')
1623             break
1624         case 'gcstress0xc':
1625         case 'gcstress0xc_zapdisable':
1626         case 'gcstress0xc_zapdisable_jitstress2':
1627         case 'gcstress0xc_zapdisable_heapverify1':
1628         case 'gcstress0xc_jitstress1':
1629         case 'gcstress0xc_jitstress2':
1630         case 'gcstress0xc_minopts_heapverify1':
1631             if (os == 'CentOS7.1') {
1632                 break
1633             }
1634             if (os == 'OSX10.12') {
1635                 // GCStress=C is currently not supported on OS X
1636                 break
1637             }
1638             if (os in bidailyCrossList) {
1639                 break
1640             }
1641             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1642                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1643                 break
1644             }
1645             assert (os == 'Windows_NT') || (os in Constants.crossList)
1646             addPeriodicTriggerHelper(job, '@weekly')
1647             break
1648
1649         case 'illink':
1650             // Testing on other operating systems TBD
1651             assert (os == 'Windows_NT' || os == 'Ubuntu')
1652             if (architecture == 'x64' || architecture == 'x86') {
1653                 if (configuration == 'Checked') {
1654                     addPeriodicTriggerHelper(job, '@daily')
1655                 }
1656             }
1657             break
1658
1659         default:
1660             println("Unknown scenario: ${scenario}");
1661             assert false
1662             break
1663     }
1664     return
1665 }
1666
1667 // **************************
1668 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1669 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1670 // builds will do a couple extra steps.
1671 // **************************
1672
1673 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1674 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1675 // it's just used for internal builds
1676 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1677 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1678     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1679
1680     if (isWindowsBuildOnlyJob) {
1681         return
1682     }
1683
1684     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1685     // Non pull request builds.
1686     if (!isPR) {
1687         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1688         return
1689     }
1690
1691     def arm64Users = [
1692         'adityamandaleeka',
1693         'AndyAyersMS',
1694         'briansull',
1695         'BruceForstall',
1696         'CarolEidt',
1697         'davidwrighton',
1698         'echesakovMSFT',
1699         'erozenfeld',
1700         'janvorli',
1701         'jashook',
1702         'pgodeq',
1703         'RussKeldorph',
1704         'sandreenko',
1705         'sdmaclea',
1706         'swaroop-sridhar',
1707         'jkotas',
1708         'markwilkie',
1709         'weshaggard'
1710     ]
1711
1712     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1713     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1714
1715     def contextString = ""
1716     def triggerString = ""
1717     def needsTrigger = true
1718     def isDefaultTrigger = false
1719     def isArm64PrivateJob = false
1720     def scenarioString = ""
1721
1722     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1723     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1724     // some jobs where the Debug configuration just does builds, no tests.
1725     //
1726     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1727     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1728     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1729     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1730
1731     if (architecture == 'armem') {
1732         assert os == 'Tizen'
1733         architecture = 'armel'
1734     }
1735
1736     switch (architecture) {
1737         case 'x64_arm64_altjit':
1738         case 'x86_arm_altjit':
1739             // TODO: for consistency, add "Build and Test" at end.
1740             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1741             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1742             break
1743
1744         case 'armel':
1745         case 'arm':
1746         case 'arm64':
1747             contextString = "${os} ${architecture} Cross ${configuration}"
1748             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1749
1750             if (scenario == 'innerloop') {
1751                 contextString += " Innerloop"
1752                 triggerString += "\\W+Innerloop"
1753             }
1754             else {
1755                 contextString += " ${scenario}"
1756                 triggerString += "\\W+${scenario}"
1757             }
1758
1759             if (configuration == 'Debug') {
1760                 contextString += " Build"
1761                 triggerString += "\\W+Build"
1762             }
1763             else {
1764                 contextString += " Build and Test"
1765                 triggerString += "\\W+Build and Test"
1766             }
1767
1768             triggerString += ".*"
1769             break
1770
1771         default:
1772             scenarioString = getScenarioDisplayString(scenario)
1773             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1774             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1775
1776             switch (scenario) {
1777                 case 'normal':
1778                     triggerString += "\\W+Build and Test.*"
1779                     break
1780
1781                 case 'corefx_innerloop': // maintain this asymmetry
1782                     triggerString += "\\W+CoreFX Tests.*"
1783                     break
1784
1785                 default:
1786                     triggerString += "\\W+${scenario}.*"
1787                     break
1788             }
1789
1790             triggerString += ".*"
1791             break
1792     }
1793
1794     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1795     // non-flow jobs that are only used as part of flow jobs.
1796
1797     switch (architecture) {
1798         case 'x64': // editor brace matching: {
1799             if (scenario == 'formatting') {
1800                 assert configuration == 'Checked'
1801                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1802                     isDefaultTrigger = true
1803                     contextString = "${os} ${architecture} Formatting"
1804                 }
1805                 break
1806             }
1807
1808             switch (os) {
1809                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1810                 case 'Debian8.4':
1811                 case 'RHEL7.2':
1812                     if (scenario == 'innerloop') {
1813                         assert !isFlowJob
1814                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1815                         isDefaultTrigger = true
1816                         break
1817                     }
1818
1819                     // fall through
1820
1821                 case 'Fedora24':
1822                 case 'Ubuntu16.04':
1823                 case 'Ubuntu16.10':
1824                     assert !isFlowJob
1825                     assert scenario != 'innerloop'
1826                     contextString = "${os} ${architecture} ${configuration} Build"
1827                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1828                     break
1829
1830                 case 'Ubuntu':
1831                     if (scenario == 'illink') {
1832                         break
1833                     }
1834                     else if (scenario == 'corefx_innerloop') {
1835                         if (configuration == 'Checked') {
1836                             isDefaultTrigger = true
1837                         }
1838                         break
1839                     }
1840
1841                     // fall through
1842
1843                 case 'OSX10.12':
1844                     // Triggers on the non-flow jobs aren't necessary here
1845                     // Corefx testing uses non-flow jobs.
1846                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1847                         needsTrigger = false
1848                         break
1849                     }
1850                     switch (scenario) {
1851                         case 'innerloop':
1852                             isDefaultTrigger = true
1853                             break
1854
1855                         case 'no_tiered_compilation_innerloop':
1856                             if (os == 'Ubuntu') {
1857                                 isDefaultTrigger = true
1858                             }
1859                             break
1860
1861                         default:
1862                             break
1863                     }
1864                     break
1865
1866                 case 'CentOS7.1':
1867                     switch (scenario) {
1868                         case 'innerloop':
1869                             // CentOS uses checked for default PR tests while debug is build only
1870                             if (configuration == 'Debug') {
1871                                 isDefaultTrigger = true
1872                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1873                                 break
1874                             }
1875
1876                             // Make sure this is a flow job to get build and test.
1877                             if (!isFlowJob) {
1878                                 needsTrigger = false
1879                                 break
1880                             }
1881
1882                             if (configuration == 'Checked') {
1883                                 assert job.name.contains("flow")
1884                                 isDefaultTrigger = true
1885                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1886                             }
1887                             break
1888
1889                         case 'normal':
1890                             // Make sure this is a flow job to get build and test.
1891                             if (!isFlowJob) {
1892                                 needsTrigger = false
1893                                 break
1894                             }
1895                             break
1896
1897                         default:
1898                             break
1899                     }
1900                     break
1901
1902                 case 'Windows_NT':
1903                     switch (scenario) {
1904                         case 'innerloop':
1905                         case 'no_tiered_compilation_innerloop':
1906                             isDefaultTrigger = true
1907                             break
1908
1909                         case 'corefx_innerloop':
1910                             if (configuration == 'Checked' || configuration == 'Release') {
1911                                 isDefaultTrigger = true
1912                             }
1913                             break
1914
1915                         default:
1916                             break
1917                     }
1918                     break
1919
1920                 default:
1921                     println("Unknown os: ${os}");
1922                     assert false
1923                     break
1924
1925             } // switch (os)
1926
1927             break
1928         // editor brace matching: }
1929
1930         case 'armel': // editor brace matching: {
1931             job.with {
1932                 publishers {
1933                     azureVMAgentPostBuildAction {
1934                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1935                     }
1936                 }
1937             }
1938
1939             switch (os) {
1940                 case 'Tizen':
1941                     if (scenario == 'innerloop') {
1942                         if (configuration == 'Checked') {
1943                             isDefaultTrigger = true
1944                         }
1945                     }
1946                     break
1947             }
1948
1949             break
1950         // editor brace matching: }
1951
1952         case 'arm':
1953         case 'arm64': // editor brace matching: {
1954
1955             switch (os) {
1956                 case 'Ubuntu':
1957                     // TODO: make arm and arm64 Ubuntu more alike
1958
1959                     if (architecture == 'arm') {
1960                         // Triggers on the non-flow jobs aren't necessary
1961                         if (!isFlowJob) {
1962                             needsTrigger = false
1963                             break
1964                         }
1965
1966                         switch (scenario) {
1967                             case 'innerloop':
1968                             case 'no_tiered_compilation_innerloop':
1969                                 if (configuration == 'Checked') {
1970                                     isDefaultTrigger = true
1971                                 }
1972                                 break
1973                         }
1974                     }
1975                     else {
1976                         assert architecture == 'arm64'
1977
1978                         switch (scenario) {
1979                             case 'innerloop':
1980                                 if (configuration == 'Debug' && !isFlowJob) {
1981                                     isDefaultTrigger = true
1982                                 }
1983                                 break
1984                         }
1985                     }
1986                     break
1987
1988                 case 'Windows_NT':
1989                     assert isArmWindowsScenario(scenario)
1990
1991                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1992                     // the non-flow Build job. All others need a trigger on the flow job.
1993                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1994                     if (isFlowJob != needsFlowJobTrigger) {
1995                         needsTrigger = false
1996                         break
1997                     }
1998
1999                     switch (scenario) {
2000                         case 'innerloop':
2001                             if (configuration == 'Debug') {
2002                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
2003                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
2004                                 // People in arm64Users will get both this and the Checked Build and Test job.
2005                                 isDefaultTrigger = true
2006                             } else if (configuration == 'Checked') {
2007                                 isDefaultTrigger = true
2008                                 isArm64PrivateJob = true
2009                             }
2010                             break
2011                         default:
2012                             isArm64PrivateJob = true
2013                             break
2014                     }
2015                     break
2016                 default:
2017                     println("NYI os: ${os}");
2018                     assert false
2019                     break
2020             }
2021             break
2022
2023         // editor brace matching: }
2024         case 'x86': // editor brace matching: {
2025             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2026             if (os == 'Ubuntu') {
2027                 // Triggers on the non-flow jobs aren't necessary here
2028                 if (!isFlowJob) {
2029                     needsTrigger = false
2030                     break
2031                 }
2032
2033                 // on-demand only for ubuntu x86
2034                 contextString = "${os} ${architecture} ${configuration} Build"
2035                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2036                 break
2037             }
2038             switch (scenario) {
2039                 case 'innerloop':
2040                 case 'no_tiered_compilation_innerloop':
2041                     isDefaultTrigger = true
2042                     break
2043                 default:
2044                     break
2045             }
2046             break
2047
2048         // editor brace matching: }
2049         case 'x64_arm64_altjit':
2050         case 'x86_arm_altjit':
2051             // Everything default
2052             break
2053
2054         default:
2055             println("Unknown architecture: ${architecture}");
2056             assert false
2057             break
2058     }
2059
2060     if (needsTrigger) {
2061         if (isArm64PrivateJob) {
2062             if (isDefaultTrigger) {
2063                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2064             }
2065             else {
2066                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2067             }
2068         }
2069         else {
2070             if (isDefaultTrigger) {
2071                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2072             }
2073             else {
2074                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2075             }
2076         }
2077     }
2078 }
2079
2080 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2081     def buildCommands = []
2082     def osGroup = getOSGroup(os)
2083     def lowerConfiguration = configuration.toLowerCase()
2084
2085     def priority = '1'
2086     if (isInnerloopTestScenario(scenario)) {
2087         priority = '0'
2088     }
2089
2090     def doCoreFxTesting = isCoreFxScenario(scenario)
2091
2092     // Calculate the build steps, archival, and xunit results
2093     switch (os) {
2094         case 'Windows_NT': // editor brace matching: {
2095             switch (architecture) {
2096                 case 'x64':
2097                 case 'x86':
2098                 case 'x86_arm_altjit':
2099                 case 'x64_arm64_altjit':
2100                     def arch = architecture
2101                     def buildOpts = ''
2102                     if (architecture == 'x86_arm_altjit') {
2103                         arch = 'x86'
2104                     }
2105                     else if (architecture == 'x64_arm64_altjit') {
2106                         arch = 'x64'
2107                     }
2108
2109                     if (scenario == 'formatting') {
2110                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2111                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2112                         break
2113                     }
2114
2115                     if (scenario == 'illink') {
2116                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2117                     }
2118
2119                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2120                     if ((lowerConfiguration == 'release') &&
2121                         (scenario in Constants.basicScenarios) &&
2122                         (architecture != 'x86_arm_altjit') &&
2123                         (architecture != 'x64_arm64_altjit')) {
2124
2125                         buildOpts += ' -enforcepgo'
2126                     }
2127
2128                     if (doCoreFxTesting) {
2129                         buildOpts += ' skiptests';
2130                     } else {
2131                         buildOpts += " -priority=${priority}"
2132                     }
2133
2134                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2135                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2136                     // 35 characters long.
2137
2138                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2139
2140                     if (!isBuildOnly) {
2141                         def runtestArguments = ''
2142                         def testOpts = 'collectdumps'
2143
2144                         if (isR2RScenario(scenario)) {
2145
2146                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2147                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2148                             // to cause the tests to be crossgen'ed.
2149
2150                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2151                                 testOpts += ' crossgenaltjit protononjit.dll'
2152                             } else {
2153                                 testOpts += ' crossgen'
2154                             }
2155
2156                             testOpts += ' runcrossgentests'
2157                         }
2158                         else if (scenario == 'jitdiff') {
2159                             testOpts += ' jitdisasm crossgen'
2160                         }
2161                         else if (scenario == 'ilrt') {
2162                             testOpts += ' ilasmroundtrip'
2163                         }
2164                         else if (isLongGc(scenario)) {
2165                             testOpts += " ${scenario} sequential"
2166                         }
2167                         else if (scenario == 'standalone_gc') {
2168                             testOpts += ' gcname clrgc.dll'
2169                         }
2170                         else if (scenario == 'illink') {
2171                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2172                         }
2173
2174                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2175                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2176                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2177                         // which is read by the xunit harness.)
2178                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2179                         {
2180                             def timeout = 1800000
2181                             testOpts += " timeout ${timeout}"
2182                         }
2183
2184                         // If we are running a stress mode, we should write out the set of key
2185                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2186
2187                         def envScriptPath = ''
2188                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2189                             def buildCommandsStr = ''
2190                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2191                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2192
2193                             if (isJitStressScenario(scenario)) {
2194                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2195                             }
2196                             else if (isR2RStressScenario(scenario)) {
2197                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2198                             }
2199
2200                             if (architecture == 'x86_arm_altjit') {
2201                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2202                             }
2203                             else if (architecture == 'x64_arm64_altjit') {
2204                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2205                             }
2206
2207                             envScriptFinalize(os, envScriptPath)
2208
2209                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2210                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2211                             // into a single script.
2212                             buildCommands += buildCommandsStr
2213                         }
2214                         else if (architecture == 'x86_arm_altjit') {
2215                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2216                         }
2217                         else if (architecture == 'x64_arm64_altjit') {
2218                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2219                         }
2220                         if (envScriptPath != '') {
2221                             testOpts += " TestEnv ${envScriptPath}"
2222                         }
2223
2224                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2225
2226                         if (doCoreFxTesting) {
2227                             if (scenario == 'corefx_innerloop') {
2228                                 // Create CORE_ROOT and testhost
2229                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2230                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2231
2232                                 // Archive and process (only) the test results
2233                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2234                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2235                             }
2236                             else {
2237                                 def workspaceRelativeFxRoot = "_/fx"
2238                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2239                                 def fxBranch = getFxBranch(branch)
2240
2241                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2242
2243                                 // Archive and process (only) the test results
2244                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2245                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2246
2247                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2248                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2249                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2250                             }
2251                         }
2252                         else if (isGcReliabilityFramework(scenario)) {
2253                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2254                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2255                         }
2256                         else {
2257                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2258                         }
2259                     } // end if (!isBuildOnly)
2260
2261                     if (!doCoreFxTesting) {
2262                         // Run the rest of the build
2263                         // Build the mscorlib for the other OS's
2264                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2265                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2266
2267                         if (arch == 'x64') {
2268                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2269                         }
2270
2271                         // Zip up the tests directory so that we don't use so much space/time copying
2272                         // 10s of thousands of files around.
2273                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2274
2275                         if (!isJitStressScenario(scenario)) {
2276                             // For Windows, pull full test results and test drops for x86/x64.
2277                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2278                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2279                         }
2280
2281                         if (scenario == 'jitdiff') {
2282                             // retrieve jit-dasm output for base commit, and run jit-diff
2283                             if (!isBuildOnly) {
2284                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2285                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2286                             }
2287                         }
2288
2289                         if (!isBuildOnly) {
2290                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2291                         }
2292                     }
2293                     break
2294                 case 'arm':
2295                 case 'arm64':
2296                     assert isArmWindowsScenario(scenario)
2297
2298                     def buildOpts = ''
2299
2300                     if (doCoreFxTesting) {
2301                         buildOpts += ' skiptests'
2302                     } else {
2303                         buildOpts += " -priority=${priority}"
2304                     }
2305
2306                     // This is now a build only job. Do not run tests. Use the flow job.
2307                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2308
2309                     if (doCoreFxTesting) {
2310                         assert isBuildOnly
2311
2312                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2313                         def envScriptPath = ''
2314                         def buildCommandsStr = ''
2315                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2316                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2317                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2318                         envScriptFinalize(os, envScriptPath)
2319                         buildCommands += buildCommandsStr
2320
2321                         def workspaceRelativeFxRootLinux = "_/fx"
2322                         def workspaceRelativeFxRootWin = "_\\fx"
2323                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2324                         def fxBranch = getFxBranch(branch)
2325
2326                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2327
2328                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2329                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2330                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2331
2332                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2333                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2334                     } else {
2335                         // Zip up the tests directory so that we don't use so much space/time copying
2336                         // 10s of thousands of files around.
2337                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2338
2339                         // Add archival.
2340                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2341                     }
2342                     break
2343                 default:
2344                     println("Unknown architecture: ${architecture}");
2345                     assert false
2346                     break
2347             }
2348             break
2349         // end case 'Windows_NT'; editor brace matching: }
2350         case 'Ubuntu':
2351         case 'Ubuntu16.04':
2352         case 'Ubuntu16.10':
2353         case 'Debian8.4':
2354         case 'OSX10.12':
2355         case 'CentOS7.1':
2356         case 'RHEL7.2':
2357         case 'Tizen':
2358         case 'Fedora24': // editor brace matching: {
2359             switch (architecture) {
2360                 case 'x64':
2361                 case 'x86':
2362                     if (architecture == 'x86' && os == 'Ubuntu') {
2363                         // build and PAL test
2364                         def dockerImage = getDockerImageName(architecture, os, true)
2365                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2366                         dockerImage = getDockerImageName(architecture, os, false)
2367                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2368                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2369                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2370                         break
2371                     }
2372
2373                     if (scenario == 'formatting') {
2374                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2375                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2376                         break
2377                     }
2378
2379                     if (scenario == 'illink') {
2380                         assert(os == 'Ubuntu')
2381                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2382                     }
2383
2384                     if (!doCoreFxTesting) {
2385                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2386                         // only on supported OS platforms.
2387                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2388                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2389
2390                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2391                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2392
2393                         // Basic archiving of the build
2394                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2395                         // And pal tests
2396                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2397                     }
2398                     else {
2399                         if (scenario == 'corefx_innerloop') {
2400                             assert os == 'Ubuntu' || 'OSX10.12'
2401                             assert architecture == 'x64'
2402
2403                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2404                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2405                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2406
2407                             break
2408                             // Archive and process (only) the test results
2409                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2410                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2411                         }
2412                         else {
2413                             // Corefx stress testing
2414                             assert os == 'Ubuntu'
2415                             assert architecture == 'x64'
2416                             assert lowerConfiguration == 'checked'
2417                             assert isJitStressScenario(scenario)
2418
2419                             // Build coreclr
2420                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2421
2422                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2423
2424                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2425                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2426                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2427                             buildCommands += envScriptCmds
2428
2429                             // Build and text corefx
2430                             def workspaceRelativeFxRoot = "_/fx"
2431                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2432                             def fxBranch = getFxBranch(branch)
2433
2434                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2435
2436                             // Archive and process (only) the test results
2437                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2438                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2439                         }
2440                     }
2441                     break
2442                 case 'armem':
2443                     // Emulator cross builds for ARM runs on Tizen currently
2444                     assert os == 'Tizen'
2445
2446                     def arm_abi = "armel"
2447                     def linuxCodeName = "tizen"
2448
2449                     // Unzip the Windows test binaries first. Exit with 0
2450                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2451
2452                     // Unpack the corefx binaries
2453                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2454                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2455
2456                     // Call the ARM CI script to cross build and test using docker
2457                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2458                     --mode=docker \\
2459                     --${arm_abi} \\
2460                     --linuxCodeName=${linuxCodeName} \\
2461                     --buildConfig=${lowerConfiguration} \\
2462                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2463                     --coreFxBinDir=./bin/CoreFxBinDir \\
2464                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2465
2466                     // Basic archiving of the build, no pal tests
2467                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2468                     break
2469                 case 'arm64':
2470                 case 'arm':
2471                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2472                     assert (os == 'Ubuntu')
2473
2474                     // Add some useful information to the log file. Ignore return codes.
2475                     buildCommands += "uname -a || true"
2476
2477                     def additionalOpts = ""
2478                     if (architecture == 'arm') {
2479                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2480                     }
2481
2482                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2483                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2484                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2485                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2486                     // ZIP up the generated CoreFX runtime and tests.
2487
2488                     def dockerImage = getDockerImageName(architecture, os, true)
2489                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2490
2491                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2492
2493                     if (doCoreFxTesting) {
2494                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2495
2496                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2497                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2498                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2499                         buildCommands += envScriptCmds
2500
2501                         // Build and text corefx
2502                         def workspaceRelativeFxRootLinux = "_/fx"
2503                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2504                         def fxBranch = getFxBranch(branch)
2505
2506                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2507
2508                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2509                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2510                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2511
2512                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2513                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2514                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2515                     }
2516                     else if (isCrossGenComparisonScenario(scenario)) {
2517                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2518
2519                         def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
2520
2521                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2522                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2523                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2524                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2525                         def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
2526                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2527
2528                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2529                         def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArchitecture}/crossgen"
2530
2531                         buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2532                         buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2533                         buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2534                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeCrossResultDir}"
2535
2536                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2537                     }
2538                     else {
2539                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2540                         // download the appropriate CoreFX packages.
2541                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2542                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2543
2544                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2545
2546                         // ZIP up for the test job (created in the flow job code):
2547                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2548                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2549                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2550                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2551
2552                         // These commands are assumed to be run from the root of the workspace.
2553                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2554                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2555
2556                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2557                     }
2558
2559                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2560                     // the cleanup task in Jenkins can't remove.
2561                     newJob.with {
2562                         publishers {
2563                             azureVMAgentPostBuildAction {
2564                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2565                             }
2566                         }
2567                     }
2568                     break
2569                 default:
2570                     println("Unknown architecture: ${architecture}");
2571                     assert false
2572                     break
2573             }
2574             break
2575         // editor brace matching: }
2576         default:
2577             println("Unknown os: ${os}");
2578             assert false
2579             break
2580     } // os
2581
2582     return buildCommands
2583 }
2584
2585 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2586 // Returns true if the job should be generated.
2587 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2588 {
2589     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2590     if (scenario == 'innerloop' && !isPR) {
2591         return false
2592     }
2593
2594     // Run basic corefx tests only on PR-triggered jobs
2595     // Runs under Release and Checked
2596     if (scenario == 'corefx_innerloop' && !isPR) {
2597         return false
2598     }
2599
2600     // Tizen is only supported for armem architecture
2601     if (os == 'Tizen' && architecture != 'armem') {
2602         return false
2603     }
2604
2605     // Filter based on architecture.
2606
2607     switch (architecture) {
2608         case 'arm64':
2609         case 'arm':
2610             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2611                 return false
2612             }
2613             break
2614         case 'armem':
2615             if (os != 'Tizen') {
2616                 return false
2617             }
2618             break
2619         case 'x86_arm_altjit':
2620         case 'x64_arm64_altjit':
2621             if (os != 'Windows_NT') {
2622                 return false
2623             }
2624             break
2625         case 'x86':
2626             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2627                 return false
2628             }
2629             break
2630         case 'x64':
2631             // Everything implemented
2632             break
2633         default:
2634             println("Unknown architecture: ${architecture}")
2635             assert false
2636             break
2637     }
2638
2639     // Which (Windows) build only jobs are required?
2640
2641     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2642
2643     if (isBuildOnly) {
2644         switch (architecture) {
2645             case 'arm':
2646             case 'arm64':
2647                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2648                 if (!isCoreFxScenario(scenario)) {
2649                     return false
2650                 }
2651                 break
2652             case 'x64':
2653             case 'x86':
2654                 if (!isNormalOrInnerloop) {
2655                     return false
2656                 }
2657                 break
2658             default:
2659                 return false
2660         }
2661     }
2662
2663     // Filter based on scenario.
2664
2665     if (isJitStressScenario(scenario)) {
2666         if (configuration != 'Checked') {
2667             return false
2668         }
2669
2670         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
2671         if (!isEnabledOS) {
2672             return false
2673         }
2674
2675         switch (architecture) {
2676             case 'x64':
2677             case 'x86_arm_altjit':
2678             case 'x64_arm64_altjit':
2679                 break
2680
2681             case 'x86':
2682                 // x86 ubuntu: no stress modes
2683                 if (os == 'Ubuntu') {
2684                     return false
2685                 }
2686                 break
2687
2688             case 'arm':
2689                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2690                 // No "regular" Windows arm corefx jobs, e.g.
2691                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2692                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2693                 if (os == 'Windows_NT') {
2694                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2695                         return false
2696                     }
2697                 }
2698                 else {
2699                     if (!isCoreFxScenario(scenario)) {
2700                         return false
2701                     }
2702                 }
2703                 break
2704
2705             case 'arm64':
2706                 if (os == 'Windows_NT') {
2707                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2708                         return false
2709                     }
2710                 }
2711                 else {
2712                     if (!isCoreFxScenario(scenario)) {
2713                         return false
2714                     }
2715                 }
2716                 break
2717
2718             default:
2719                 // arm64: stress is handled through flow jobs.
2720                 // armem: no stress jobs for ARM emulator.
2721                 return false
2722         }
2723     }
2724     else if (isR2RScenario(scenario)) {
2725         if (os != 'Windows_NT') {
2726             return false
2727         }
2728
2729         if (isR2RBaselineScenario(scenario)) {
2730             // no need for Debug scenario; Checked is sufficient
2731             if (configuration != 'Checked' && configuration != 'Release') {
2732                 return false
2733             }
2734         }
2735         else if (isR2RStressScenario(scenario)) {
2736             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2737             if (configuration != 'Checked') {
2738                 return false
2739             }
2740         }
2741
2742         switch (architecture) {
2743             case 'arm':
2744             case 'arm64':
2745                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2746                 return false
2747
2748             default:
2749                 break
2750         }
2751     }
2752     else if (isCrossGenComparisonScenario(scenario)) {
2753         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2754     }
2755     else {
2756         // Skip scenarios
2757         switch (scenario) {
2758             case 'ilrt':
2759                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2760                 // the default scenario build
2761                 if (os != 'Windows_NT') {
2762                     return false
2763                 }
2764                 // Only x64 for now
2765                 if (architecture != 'x64') {
2766                     return false
2767                 }
2768                 // Release only
2769                 if (configuration != 'Release') {
2770                     return false
2771                 }
2772                 break
2773             case 'jitdiff':
2774                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2775                     return false
2776                 }
2777                 if (architecture != 'x64') {
2778                     return false
2779                 }
2780                 if (configuration != 'Checked') {
2781                     return false
2782                 }
2783                 break
2784             case 'longgc':
2785             case 'gcsimulator':
2786                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2787                     return false
2788                 }
2789                 if (architecture != 'x64') {
2790                     return false
2791                 }
2792                 if (configuration != 'Release') {
2793                     return false
2794                 }
2795                 break
2796             case 'gc_reliability_framework':
2797             case 'standalone_gc':
2798                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2799                     return false
2800                 }
2801
2802                 if (architecture != 'x64') {
2803                     return false
2804                 }
2805
2806                 if (configuration != 'Release' && configuration != 'Checked') {
2807                     return false
2808                 }
2809                 break
2810             // We only run Windows and Ubuntu x64 Checked for formatting right now
2811             case 'formatting':
2812                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2813                     return false
2814                 }
2815                 if (architecture != 'x64') {
2816                     return false
2817                 }
2818                 if (configuration != 'Checked') {
2819                     return false
2820                 }
2821                 break
2822             case 'illink':
2823                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2824                     return false
2825                 }
2826                 if (architecture != 'x64' && architecture != 'x86') {
2827                     return false
2828                 }
2829                 break
2830             case 'normal':
2831                 // Nothing skipped
2832                 break
2833             case 'innerloop':
2834                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2835                     return false
2836                 }
2837                 break
2838             case 'corefx_innerloop':
2839                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2840                     return false
2841                 }
2842                 if (architecture != 'x64') {
2843                     return false
2844                 }
2845                 break
2846             default:
2847                 println("Unknown scenario: ${scenario}")
2848                 assert false
2849                 break
2850         }
2851     }
2852
2853     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2854     // runtime behavior, not compile-time behavior, are not interesting.
2855     switch (architecture) {
2856         case 'x86_arm_altjit':
2857         case 'x64_arm64_altjit':
2858             if (isGCStressRelatedTesting(scenario)) {
2859                 return false
2860             }
2861             break
2862         default:
2863             break
2864     }
2865
2866     // The job was not filtered out, so we should generate it!
2867     return true
2868 }
2869
2870 Constants.allScenarios.each { scenario ->
2871     [true, false].each { isPR ->
2872         Constants.architectureList.each { architecture ->
2873             Constants.configurationList.each { configuration ->
2874                 Constants.osList.each { os ->
2875                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2876                     // and reset the os to Windows_NT
2877                     def isBuildOnly = false
2878                     if (os == 'Windows_NT_BuildOnly') {
2879                         isBuildOnly = true
2880                         os = 'Windows_NT'
2881                     }
2882
2883                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2884                         return
2885                     }
2886
2887                     // Calculate names
2888                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2889                     def folderName = getJobFolder(scenario)
2890
2891                     // Create the new job
2892                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2893
2894                     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
2895                     // "top-level" jobs. Build only jobs are such jobs.
2896                     if (!isBuildOnly)
2897                     {
2898                         addToViews(newJob, isPR, architecture, os)
2899                     }
2900
2901                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2902
2903                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2904                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2905                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2906
2907                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2908                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2909                     // artifacts produced.
2910                     if ((architecture == 'armem') && (os == 'Tizen')) {
2911                         // Define the Windows Tests and Corefx build job names
2912                         def lowerConfiguration = configuration.toLowerCase()
2913                         def WindowsTestsName = projectFolder + '/' +
2914                                                Utilities.getFullJobName(project,
2915                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2916                                                                         false)
2917                         def fxBranch = getFxBranch(branch)
2918                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2919                                            Utilities.getFolderName(fxBranch)
2920
2921                         def arm_abi = 'armel'
2922                         def corefx_os = 'tizen'
2923
2924                         // Let's use release CoreFX to test checked CoreCLR,
2925                         // because we do not generate checked CoreFX in CoreFX CI yet.
2926                         def corefx_lowerConfiguration = lowerConfiguration
2927                         if (lowerConfiguration == 'checked') {
2928                             corefx_lowerConfiguration = 'release'
2929                         }
2930
2931                         // Copy the Windows test binaries and the Corefx build binaries
2932                         newJob.with {
2933                             steps {
2934                                 copyArtifacts(WindowsTestsName) {
2935                                     includePatterns('bin/tests/tests.zip')
2936                                     buildSelector {
2937                                         latestSuccessful(true)
2938                                     }
2939                                 }
2940                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2941                                     includePatterns('bin/build.tar.gz')
2942                                     buildSelector {
2943                                         latestSuccessful(true)
2944                                     }
2945                                 }
2946                             } // steps
2947                         } // newJob.with
2948                     }
2949
2950                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2951
2952                     newJob.with {
2953                         steps {
2954                             if (os == 'Windows_NT') {
2955                                 buildCommands.each { buildCommand ->
2956                                     batchFile(buildCommand)
2957                                 }
2958                             }
2959                             else {
2960                                 buildCommands.each { buildCommand ->
2961                                     shell(buildCommand)
2962                                 }
2963                             }
2964                         } // steps
2965                     } // newJob.with
2966
2967                 } // os
2968             } // configuration
2969         } // architecture
2970     } // isPR
2971 } // scenario
2972
2973 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2974 // Returns the newly created job.
2975 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2976 {
2977     def osGroup = getOSGroup(os)
2978     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2979
2980     def jobFolder = getJobFolder(scenario)
2981     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2982         parameters {
2983             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2984         }
2985
2986         steps {
2987             // Set up the copies
2988
2989             // Coreclr build we are trying to test
2990             //
2991             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2992
2993             copyArtifacts(inputCoreCLRBuildName) {
2994                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2995                 buildSelector {
2996                     buildNumber('${CORECLR_BUILD}')
2997                 }
2998             }
2999
3000             if (isCoreFxScenario(scenario)) {
3001
3002                 // Only arm/arm64 supported for corefx testing now.
3003                 assert architecture == 'arm' || architecture == 'arm64'
3004
3005                 // Unzip CoreFx runtime
3006                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3007
3008                 // Unzip CoreFx tests.
3009                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3010
3011                 // Add the script to run the corefx tests
3012                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3013                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
3014                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3015                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3016
3017             } else { // !isCoreFxScenario(scenario)
3018
3019                 // Unzip tests.
3020                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3021
3022                 def buildCommands = ""
3023
3024                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3025                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3026                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3027
3028                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3029                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3030
3031                 // For all jobs
3032                 addEnvVariable("CORE_ROOT", coreRootLocation)
3033                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3034                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3035
3036                 // If we are running a stress mode, we'll set those variables as well
3037                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3038                     def stressValues = null
3039                     if (isJitStressScenario(scenario)) {
3040                         stressValues = Constants.jitStressModeScenarios[scenario]
3041                     }
3042                     else {
3043                         stressValues = Constants.r2rStressScenarios[scenario]
3044                     }
3045
3046                     stressValues.each { key, value ->
3047                         addEnvVariable(key, value)
3048                     }
3049                 }
3050
3051                 if (isR2RScenario(scenario)) {
3052                     // Crossgen the framework assemblies.
3053                     buildCommands += """
3054 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3055 @goto skip_PrecompileAssembly
3056
3057 :PrecompileAssembly
3058 @REM Skip mscorlib since it is already precompiled.
3059 @if /I "%3" == "mscorlib.dll" exit /b 0
3060 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3061
3062 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3063 @if "%errorlevel%" == "-2146230517" (
3064     echo %2 is not a managed assembly.
3065 ) else if "%errorlevel%" == "-2146234344" (
3066     echo %2 is not a managed assembly.
3067 ) else if %errorlevel% neq 0 (
3068     echo Unable to precompile %2
3069 ) else (
3070     echo Precompiled %2
3071 )
3072 @exit /b 0
3073
3074 :skip_PrecompileAssembly
3075 """
3076
3077                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3078                     // crossgen on tests before running them.
3079                     addEnvVariable("RunCrossGen", "true")
3080                 } // isR2RScenario(scenario)
3081
3082                 // Create the smarty command
3083                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3084                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3085                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3086                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3087
3088                 // Exclude tests based on scenario.
3089                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3090                     addArchSpecificExclude(architecture, excludeTag)
3091                 }
3092
3093                 if (isInnerloopTestScenario(scenario)) {
3094                     addExclude("pri1")
3095                 }
3096
3097                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3098                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3099                 addExclude("LONG_RUNNING")
3100
3101                 smartyCommand += "/lstFile Tests.lst"
3102
3103                 def testListArch = [
3104                     'arm64': 'arm64',
3105                     'arm': 'arm'
3106                 ]
3107
3108                 def archLocation = testListArch[architecture]
3109
3110                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3111                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3112                 addCommand("${smartyCommand}")
3113
3114                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3115                 // However, we also need to remove all the variables that were set during this batch file, so we
3116                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3117                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3118                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3119                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3120                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3121                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3122
3123                 // ZIP up the smarty output, no matter what the smarty result.
3124                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3125
3126                 addCommand("echo %errorlevel%")
3127                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3128
3129                 // Use the smarty errorlevel as the script errorlevel.
3130                 addCommand("exit /b %__save_smarty_errorlevel%")
3131
3132                 batchFile(buildCommands)
3133             } // non-corefx testing
3134         } // steps
3135     } // job
3136
3137     if (!isCoreFxScenario(scenario)) {
3138         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3139
3140         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3141         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3142         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3143         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3144     }
3145
3146     return newJob
3147 }
3148
3149 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3150 // E.g., non-Windows tests.
3151 // Returns the newly created job.
3152 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3153 {
3154     def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
3155     def doCoreFxTesting = isCoreFxScenario(scenario)
3156
3157     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3158
3159     def osGroup = getOSGroup(os)
3160     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3161
3162     def testOpts = ''
3163     def useServerGC = false
3164
3165     // Enable Server GC for Ubuntu PR builds
3166     // REVIEW: why? Does this apply to all architectures? Why only PR?
3167     if (os == 'Ubuntu' && isPR) {
3168         testOpts += ' --useServerGC'
3169         useServerGC = true
3170     }
3171
3172     if (isR2RScenario(scenario)) {
3173
3174         testOpts += ' --crossgen --runcrossgentests'
3175
3176         if (scenario == 'r2r_jitstress1') {
3177             testOpts += ' --jitstress=1'
3178         }
3179         else if (scenario == 'r2r_jitstress2') {
3180             testOpts += ' --jitstress=2'
3181         }
3182         else if (scenario == 'r2r_jitstress1_tiered') {
3183             testOpts += ' --jitstress=1'
3184         }
3185         else if (scenario == 'r2r_jitstress2_tiered') {
3186             testOpts += ' --jitstress=2'
3187         }
3188         else if (scenario == 'r2r_jitstressregs1') {
3189             testOpts += ' --jitstressregs=1'
3190         }
3191         else if (scenario == 'r2r_jitstressregs2') {
3192             testOpts += ' --jitstressregs=2'
3193         }
3194         else if (scenario == 'r2r_jitstressregs3') {
3195             testOpts += ' --jitstressregs=3'
3196         }
3197         else if (scenario == 'r2r_jitstressregs4') {
3198             testOpts += ' --jitstressregs=4'
3199         }
3200         else if (scenario == 'r2r_jitstressregs8') {
3201             testOpts += ' --jitstressregs=8'
3202         }
3203         else if (scenario == 'r2r_jitstressregs0x10') {
3204             testOpts += ' --jitstressregs=0x10'
3205         }
3206         else if (scenario == 'r2r_jitstressregs0x80') {
3207             testOpts += ' --jitstressregs=0x80'
3208         }
3209         else if (scenario == 'r2r_jitstressregs0x1000') {
3210             testOpts += ' --jitstressregs=0x1000'
3211         }
3212         else if (scenario == 'r2r_jitminopts') {
3213             testOpts += ' --jitminopts'
3214         }
3215         else if (scenario == 'r2r_jitforcerelocs') {
3216             testOpts += ' --jitforcerelocs'
3217         }
3218         else if (scenario == 'r2r_gcstress15') {
3219             testOpts += ' --gcstresslevel=0xF'
3220         }
3221     }
3222     else if (scenario == 'jitdiff') {
3223         testOpts += ' --jitdisasm --crossgen'
3224     }
3225     else if (scenario == 'illink') {
3226         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3227     }
3228     else if (isLongGc(scenario)) {
3229         // Long GC tests behave very poorly when they are not
3230         // the only test running (many of them allocate until OOM).
3231         testOpts += ' --sequential'
3232
3233         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3234         // for running long GC and GCSimulator tests, respectively. We don't use them
3235         // here because using a playlist file produces much more readable output on the CI machines
3236         // and reduces running time.
3237         //
3238         // The Long GC playlist contains all of the tests that are
3239         // going to be run. The GCSimulator playlist contains all of
3240         // the GC simulator tests.
3241         if (scenario == 'longgc') {
3242             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3243         }
3244         else if (scenario == 'gcsimulator') {
3245             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3246         }
3247     }
3248     else if (isGcReliabilityFramework(scenario)) {
3249         testOpts += ' --build-overlay-only'
3250     }
3251     else if (scenario == 'standalone_gc') {
3252         if (osGroup == 'OSX') {
3253             testOpts += ' --gcname=libclrgc.dylib'
3254         }
3255         else if (osGroup == 'Linux') {
3256             testOpts += ' --gcname=libclrgc.so'
3257         }
3258         else {
3259             println("Unexpected OS group: ${osGroup} for os ${os}")
3260             assert false
3261         }
3262     }
3263
3264     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3265     // will be null in this case.
3266
3267     def jobFolder = getJobFolder(scenario)
3268     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3269         parameters {
3270             if (inputTestsBuildName != null) {
3271                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3272             }
3273             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3274         }
3275
3276         steps {
3277             // Set up the copies
3278
3279             // Coreclr build containing the tests and mscorlib
3280             // pri1 jobs still need to copy windows_nt built tests
3281             if (inputTestsBuildName != null) {
3282                 copyArtifacts(inputTestsBuildName) {
3283                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3284                     buildSelector {
3285                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3286                     }
3287                 }
3288             }
3289
3290             // Coreclr build we are trying to test
3291             //
3292             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3293
3294             copyArtifacts(inputCoreCLRBuildName) {
3295                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3296                 buildSelector {
3297                     buildNumber('${CORECLR_BUILD}')
3298                 }
3299             }
3300
3301             if (isUbuntuArmJob) {
3302                 // Add some useful information to the log file. Ignore return codes.
3303                 shell("uname -a || true")
3304             }
3305
3306             if (architecture == 'x86') {
3307                 shell("mkdir ./bin/CoreFxNative")
3308
3309                 def fxBranch = getFxBranch(branch)
3310                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3311
3312                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3313                     includePatterns('bin/build.tar.gz')
3314                     targetDirectory('bin/CoreFxNative')
3315                     buildSelector {
3316                         latestSuccessful(true)
3317                     }
3318                 }
3319
3320                 shell("mkdir ./bin/CoreFxBinDir")
3321                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3322             }
3323
3324             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3325             if (doCoreFxTesting) {
3326                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3327                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3328             }
3329             else {
3330                 // Unzip the tests first.  Exit with 0
3331                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3332                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3333             }
3334
3335             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3336             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3337             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3338             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3339             // copied correctly.
3340             if (!doCoreFxTesting) {
3341                 if (isUbuntuArmJob) {
3342                     def lowerConfiguration = configuration.toLowerCase()
3343                     shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3344                     shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3345                 }
3346                 else {
3347                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3348                 }
3349             }
3350
3351             // Execute the tests
3352             def runDocker = isNeedDocker(architecture, os, false)
3353             def dockerPrefix = ""
3354             def dockerCmd = ""
3355             if (runDocker) {
3356                 def dockerImage = getDockerImageName(architecture, os, false)
3357                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3358                 dockerCmd = dockerPrefix + "${dockerImage} "
3359             }
3360
3361             // If we are running a stress mode, we'll set those variables first.
3362             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3363             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3364                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3365                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3366                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3367                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3368                 shell("${envScriptCmds}")
3369                 testOpts += " --test-env=${scriptFileName}"
3370             }
3371
3372             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3373             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3374             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3375             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3376             if (isGCStressRelatedTesting(scenario)) {
3377                 if (architecture == 'x64') {
3378                     shell('./init-tools.sh')
3379                 }
3380             }
3381
3382             if (doCoreFxTesting) {
3383                 shell("""\
3384 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3385             }
3386             else {
3387                 def runScript = "${dockerCmd}./tests/runtest.sh"
3388
3389                 shell("""\
3390 ${runScript} \\
3391     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3392     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3393     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3394     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3395             }
3396
3397             if (isGcReliabilityFramework(scenario)) {
3398                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3399                 if (useServerGC) {
3400                     if (runDocker) {
3401                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3402                     }
3403                     else {
3404                         shell("export COMPlus_gcServer=1")
3405                     }
3406                 }
3407
3408                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3409             }
3410         } // steps
3411     } // job
3412
3413     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3414     if (os in ['Ubuntu']) {
3415         SummaryBuilder summaries = new SummaryBuilder()
3416         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3417         summaries.emit(newJob)
3418     }
3419
3420     if (doCoreFxTesting) {
3421         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3422         if ((os == "Ubuntu") && (architecture == 'arm')) {
3423             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3424             //
3425             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3426             //
3427             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3428             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3429         }
3430         else {
3431             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3432         }
3433     }
3434     else {
3435         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3436     }
3437
3438     return newJob
3439 }
3440
3441 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3442 {
3443     assert isCrossGenComparisonScenario(scenario)
3444
3445     def osGroup = getOSGroup(os)
3446     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3447
3448     def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
3449     def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
3450     def workspaceRelativeNativeResultDir = "_/${osGroup}.${architecture}_${architecture}.${configuration}"
3451
3452     def jobFolder = getJobFolder(scenario)
3453     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3454         parameters {
3455             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3456         }
3457
3458         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3459
3460         steps {
3461             copyArtifacts(inputCoreCLRBuildName) {
3462                 includePatterns("${workspaceRelativeArtifactsArchive}")
3463                 buildSelector {
3464                     buildNumber('${CORECLR_BUILD}')
3465                 }
3466             }
3467
3468             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3469
3470             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3471             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3472             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3473             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3474
3475             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3476             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3477
3478             shell("mkdir -p ${workspaceRelativeNativeResultDir}")
3479             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
3480             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
3481             shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}")
3482         } // steps
3483     }  // job
3484
3485     Utilities.addArchival(newJob, "${workspaceRelativeCrossResultDir}/**")
3486     Utilities.addArchival(newJob, "${workspaceRelativeNativeResultDir}/**")
3487
3488     return newJob
3489 }
3490
3491 // Create a test job that will be used by a flow job.
3492 // Returns the newly created job.
3493 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3494 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3495 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3496 {
3497     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3498
3499     def newJob = null
3500     if (windowsArmJob) {
3501         assert inputTestsBuildName == null
3502         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3503     }
3504     else if (isCrossGenComparisonScenario(scenario)) {
3505         assert inputTestsBuildName == null
3506         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3507     }
3508     else {
3509         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3510     }
3511
3512     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3513
3514     if (scenario == 'jitdiff') {
3515         def osGroup = getOSGroup(os)
3516         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3517     }
3518
3519     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3520     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3521
3522     return newJob
3523 }
3524
3525 // Create a flow job to tie together a build job with the given test job.
3526 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3527 // Returns the new flow job.
3528 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3529 {
3530     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3531     // Linux CoreCLR test
3532     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3533     def jobFolder = getJobFolder(scenario)
3534
3535     def newFlowJob = null
3536
3537     if (inputTestsBuildName == null) {
3538         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3539                         buildFlow("""\
3540 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3541
3542 // And then build the test build
3543 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3544 """)
3545         }
3546         JobReport.Report.addReference(inputCoreCLRBuildName)
3547         JobReport.Report.addReference(fullTestJobName)
3548     }
3549     else {
3550         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3551                         buildFlow("""\
3552 // Build the input jobs in parallel
3553 parallel (
3554 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3555 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3556 )
3557
3558 // And then build the test build
3559 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3560                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3561 """)
3562         }
3563         JobReport.Report.addReference(inputCoreCLRBuildName)
3564         JobReport.Report.addReference(inputTestsBuildName)
3565         JobReport.Report.addReference(fullTestJobName)
3566     }
3567
3568     addToViews(newFlowJob, isPR, architecture, os)
3569
3570     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3571
3572     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3573     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3574
3575     return newFlowJob
3576 }
3577
3578 // Determine if we should generate a flow job for the given parameters.
3579 // Returns true if the job should be generated.
3580 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3581 {
3582     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3583     if (scenario == 'innerloop' && !isPR) {
3584         return false
3585     }
3586
3587     if (scenario == 'corefx_innerloop') {
3588         return false
3589     }
3590
3591     // Filter based on OS and architecture.
3592
3593     switch (architecture) {
3594         case 'arm64':
3595         case 'arm':
3596             if (os != "Ubuntu" && os != "Windows_NT") {
3597                 return false
3598             }
3599             break
3600         case 'x86':
3601             if (os != "Ubuntu") {
3602                 return false
3603             }
3604             break
3605         case 'x64':
3606             if (!(os in Constants.crossList)) {
3607                 return false
3608             }
3609             if (os == "Windows_NT") {
3610                 return false
3611             }
3612             break
3613         case 'armem':
3614         case 'x86_arm_altjit':
3615         case 'x64_arm64_altjit':
3616             // No flow jobs
3617             return false
3618         default:
3619             println("Unknown architecture: ${architecture}")
3620             assert false
3621             break
3622     }
3623
3624     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3625
3626     // Filter based on scenario in OS.
3627
3628     if (os == 'Windows_NT') {
3629         assert architecture == 'arm' || architecture == 'arm64'
3630         if (!isArmWindowsScenario(scenario)) {
3631             return false
3632         }
3633         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3634             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3635             // To do that, it doesn't require a flow job.
3636             return false
3637         }
3638     }
3639     else {
3640         // Non-Windows
3641         if (architecture == 'arm64') {
3642             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3643                 return false
3644             }
3645
3646             if (isNormalOrInnerloop && (configuration == 'Debug')) {
3647                 // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3648                 // To do that, it doesn't require a flow job.
3649                 return false
3650             }
3651         }
3652         else if (architecture == 'arm') {
3653             if (!(scenario in Constants.validLinuxArmScenarios)) {
3654                 return false
3655             }
3656         }
3657         else if (architecture == 'x86') {
3658             // Linux/x86 only want innerloop and default test
3659             if (!isNormalOrInnerloop) {
3660                 return false
3661             }
3662         }
3663         else if (architecture == 'x64') {
3664             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3665             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3666             // build from a Windows machine.
3667             if (isCoreFxScenario(scenario)) {
3668                 return false
3669             }
3670         }
3671     }
3672
3673     // For CentOS, we only want Checked/Release builds.
3674     if (os == 'CentOS7.1') {
3675         if (configuration != 'Checked' && configuration != 'Release') {
3676             return false
3677         }
3678         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3679             return false
3680         }
3681     }
3682
3683     // For RedHat and Debian, we only do Release builds.
3684     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3685         if (configuration != 'Release') {
3686             return false
3687         }
3688         if (!isNormalOrInnerloop) {
3689             return false
3690         }
3691     }
3692
3693     // Next, filter based on scenario.
3694
3695     if (isJitStressScenario(scenario)) {
3696         if (configuration != 'Checked') {
3697             return false
3698         }
3699     }
3700     else if (isR2RBaselineScenario(scenario)) {
3701         if (configuration != 'Checked' && configuration != 'Release') {
3702             return false
3703         }
3704     }
3705     else if (isR2RStressScenario(scenario)) {
3706         if (configuration != 'Checked') {
3707             return false
3708         }
3709     }
3710     else if (isCrossGenComparisonScenario(scenario)) {
3711         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3712     }
3713     else {
3714         // Skip scenarios
3715         switch (scenario) {
3716             case 'ilrt':
3717             case 'longgc':
3718             case 'gcsimulator':
3719                 // Long GC tests take a long time on non-Release builds
3720                 // ilrt is also Release only
3721                 if (configuration != 'Release') {
3722                     return false
3723                 }
3724                 break
3725
3726             case 'jitdiff':
3727                 if (configuration != 'Checked') {
3728                     return false
3729                 }
3730                 break
3731
3732             case 'gc_reliability_framework':
3733             case 'standalone_gc':
3734                 if (configuration != 'Release' && configuration != 'Checked') {
3735                     return false
3736                 }
3737                 break
3738
3739             case 'formatting':
3740                 return false
3741
3742             case 'illink':
3743                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3744                     return false
3745                 }
3746                 break
3747
3748             case 'normal':
3749                 // Nothing skipped
3750                 break
3751
3752             case 'innerloop':
3753                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3754                     return false
3755                 }
3756                 break
3757
3758             default:
3759                 println("Unknown scenario: ${scenario}")
3760                 assert false
3761                 break
3762         }
3763     }
3764
3765     // The job was not filtered out, so we should generate it!
3766     return true
3767 }
3768
3769 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3770 Constants.allScenarios.each { scenario ->
3771     [true, false].each { isPR ->
3772         Constants.architectureList.each { architecture ->
3773             Constants.configurationList.each { configuration ->
3774                 Constants.osList.each { os ->
3775
3776                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3777                         return
3778                     }
3779
3780                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3781                     def doCoreFxTesting = isCoreFxScenario(scenario)
3782                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3783
3784                     // Figure out the job name of the CoreCLR build the test will depend on.
3785
3786                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3787                     def inputCoreCLRBuildIsBuildOnly = false
3788                     if (doCoreFxTesting) {
3789                         // Every CoreFx test depends on its own unique build.
3790                         inputCoreCLRBuildScenario = scenario
3791                         if (windowsArmJob) {
3792                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3793                             inputCoreCLRBuildIsBuildOnly = true
3794                         }
3795                     }
3796                     if (doCrossGenComparison) {
3797                         inputCoreCLRBuildScenario = scenario
3798                     }
3799
3800                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3801                     def inputCoreCLRBuildName = projectFolder + '/' +
3802                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3803
3804                     // Figure out the name of the build job that the test job will depend on.
3805                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3806                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3807                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3808                     // CoreFX build creates the tests.
3809
3810                     def inputTestsBuildName = null
3811
3812                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison) {
3813                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3814
3815                         def inputTestsBuildArch = architecture
3816                         if (architecture == "arm64") {
3817                             // Use the x64 test build for arm64 unix
3818                             inputTestsBuildArch = "x64"
3819                         }
3820                         else if (architecture == "arm") {
3821                             // Use the x86 test build for arm unix
3822                             inputTestsBuildArch = "x86"
3823                         }
3824
3825                         def inputTestsBuildIsBuildOnly = true
3826
3827                         inputTestsBuildName = projectFolder + '/' +
3828                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3829                     }
3830
3831                     // =============================================================================================
3832                     // Create the test job
3833                     // =============================================================================================
3834
3835                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3836
3837                     // =============================================================================================
3838                     // Create a build flow to join together the build and tests required to run this test.
3839                     // =============================================================================================
3840
3841                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3842                         // Do not create the flow job for RHEL jobs.
3843                         return
3844                     }
3845
3846                     def fullTestJobName = projectFolder + '/' + testJob.name
3847                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3848
3849                 } // os
3850             } // configuration
3851         } // architecture
3852     } // isPR
3853 } // scenario
3854
3855 JobReport.Report.generateJobReport(out)
3856
3857 // Make the call to generate the help job
3858 Utilities.createHelperJob(this, project, branch,
3859     "Welcome to the ${project} Repository",  // This is prepended to the help message
3860     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3861
3862 Utilities.addCROSSCheck(this, project, branch)