netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu'      : 'Linux',
  21                       'Ubuntu16.04' : 'Linux',
  22                       'Ubuntu16.10' : 'Linux',
  23                       'RHEL7.2'     : 'Linux',
  24                       'Debian8.4'   : 'Linux',
  25                       'Fedora24'    : 'Linux',
  26                       'CentOS7.1'   : 'Linux',
  27                       'Tizen'       : 'Linux',
  28                       'OSX10.12'    : 'OSX',
  29                       'Windows_NT'  : 'Windows_NT']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitectures(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return ['x86','x64']
  39
  40         case 'arm64':
  41             return ['x64']
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  51     def static LimitedHardwareBranches = [
  52                'master']
  53
  54     // Innerloop build OS's
  55     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  56     // test execution in the build flow runs.  It generates the exact same build
  57     // as Windows_NT but without running the tests.
  58     def static osList = [
  59                'Ubuntu',
  60                'Debian8.4',
  61                'OSX10.12',
  62                'Windows_NT',
  63                'Windows_NT_BuildOnly',
  64                'CentOS7.1',
  65                'RHEL7.2',
  66                'Ubuntu16.04',
  67                'Ubuntu16.10',
  68                'Fedora24',
  69                'Tizen']
  70
  71     def static crossList = [
  72                'Ubuntu',
  73                'Debian8.4',
  74                'OSX10.12',
  75                'Windows_NT',
  76                'CentOS7.1',
  77                'RHEL7.2']
  78
  79     // This is a set of JIT stress modes combined with the set of variables that
  80     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  81     // the values are the environment variables
  82     def static jitStressModeScenarios = [
  83                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  84                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  85                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  86                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  87                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  88                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  89                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  90                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  91                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  92                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  93                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  94                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  95                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  96                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  97                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  98                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
  99                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 100                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 101                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 102                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 103                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 104                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 105                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 106                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 107                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 108                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 109                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 110                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 111                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 112                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 113                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 114                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 115                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 116                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 117                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 118                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 119                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 120                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 121                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 122                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 123                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 124                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 125                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 126                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 127                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 128                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 129                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 130                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 131                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 132                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 133                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 134                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 135                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 136                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 137                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 138                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 139     ]
 140
 141     // This is a set of ReadyToRun stress scenarios
 142     def static r2rStressScenarios = [
 143                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 144                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 145                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 146                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 147                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 148                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 149                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 150                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 151                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 152                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 153                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 154                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 155                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 156                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 157                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 158                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 159     ]
 160
 161     // This is the basic set of scenarios
 162     def static basicScenarios = [
 163                'innerloop',
 164                'normal',
 165                'ilrt',
 166                'r2r',
 167                'longgc',
 168                'formatting',
 169                'gcsimulator',
 170                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 171                'standalone_gc',
 172                'gc_reliability_framework',
 173                'illink',
 174                'corefx_innerloop',
 175                'crossgen_comparison']
 176
 177     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 178
 179     // Valid PR trigger combinations.
 180     def static prTriggeredValidInnerLoopCombos = [
 181         'Windows_NT': [
 182             'x64': [
 183                 'Checked'
 184             ],
 185             'x86': [
 186                 'Checked',
 187                 'Release'
 188             ],
 189             'arm': [
 190                 'Debug',
 191                 'Checked'
 192             ],
 193             'arm64': [
 194                 'Debug',
 195                 'Checked'
 196             ]
 197         ],
 198         'Windows_NT_BuildOnly': [
 199             'x64': [
 200                 'Checked',
 201                 'Release'
 202             ],
 203             'x86': [
 204                 'Checked',
 205                 'Release'
 206             ],
 207             'arm': [
 208                 'Checked'
 209             ],
 210         ],
 211         'Ubuntu': [
 212             'x64': [
 213                 'Checked'
 214             ],
 215             'arm': [
 216                 'Checked'
 217             ]
 218         ],
 219         'Ubuntu16.04': [
 220             'arm64': [
 221                 'Checked'
 222             ]
 223         ],
 224         'CentOS7.1': [
 225             'x64': [
 226                 'Debug',
 227                 'Checked'
 228             ]
 229         ],
 230         'OSX10.12': [
 231             'x64': [
 232                 'Checked'
 233             ]
 234         ],
 235         'Tizen': [
 236             'armem': [
 237                 'Checked'
 238             ]
 239         ]
 240     ]
 241
 242     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 243     // to Tests.lst file categories to exclude.
 244     //
 245     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 246     // and with the same values, with some commented out, for easier maintenance.
 247     //
 248     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 249     //
 250     def static validArmWindowsScenarios = [
 251                'innerloop':                              [],
 252                'normal':                                 [],
 253                // 'ilrt'
 254                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 255                // 'longgc'
 256                // 'formatting'
 257                // 'gcsimulator'
 258                // 'jitdiff'
 259                // 'standalone_gc'
 260                // 'gc_reliability_framework'
 261                // 'illink'
 262                // 'corefx_innerloop'
 263                // 'crossgen_comparison'
 264                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 277                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 278                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 279                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 280                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 281                'tieredcompilation':                      [],
 282                'no_tiered_compilation':                  [],
 283                'no_tiered_compilation_innerloop':        [],
 284                'forcerelocs':                            [],
 285                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 290                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 293                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 294                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 306                // 'jitsse2only'                          // Only relevant to xarch
 307                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 308                // 'jitincompletehwintrinsic'
 309                // 'jitx86hwintrinsicnoavx'
 310                // 'jitx86hwintrinsicnoavx2'
 311                // 'jitx86hwintrinsicnosimd'
 312                // 'jitnox86hwintrinsic'
 313                'corefx_baseline':                        [],    // corefx tests don't use smarty
 314                'corefx_minopts':                         [],    // corefx tests don't use smarty
 315                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 316                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 317                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 318                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 319                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 320                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 321                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 322                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 323                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 324                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 325                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 326                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 327                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 328                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 329                'heapverify1':                            [],
 330                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 331                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 332                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 333                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 334                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 336
 337                //
 338                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 339                //
 340
 341                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 342                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 343                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 344                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 345                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 346                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 347                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 348                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 349                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 350                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 351                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 352                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 353                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 354                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 355                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 356                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 357                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 358                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 359                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 360     ]
 361
 362     def static validLinuxArmScenarios = [
 363                'innerloop',
 364                'normal',
 365                // 'ilrt'
 366                'r2r',
 367                // 'longgc'
 368                // 'formatting'
 369                // 'gcsimulator'
 370                // 'jitdiff'
 371                // 'standalone_gc'
 372                // 'gc_reliability_framework'
 373                // 'illink'
 374                // 'corefx_innerloop'
 375                'crossgen_comparison',
 376                'r2r_jitstress1',
 377                'r2r_jitstress2',
 378                'r2r_jitstress1_tiered',
 379                'r2r_jitstress2_tiered',
 380                'r2r_jitstressregs1',
 381                'r2r_jitstressregs2',
 382                'r2r_jitstressregs3',
 383                'r2r_jitstressregs4',
 384                'r2r_jitstressregs8',
 385                'r2r_jitstressregs0x10',
 386                'r2r_jitstressregs0x80',
 387                'r2r_jitstressregs0x1000',
 388                'r2r_jitminopts',
 389                'r2r_jitforcerelocs',
 390                'r2r_gcstress15',
 391                'r2r_no_tiered_compilation',
 392                'minopts',
 393                'tieredcompilation',
 394                'no_tiered_compilation',
 395                'no_tiered_compilation_innerloop',
 396                'forcerelocs',
 397                'jitstress1',
 398                'jitstress2',
 399                'jitstress1_tiered',
 400                'jitstress2_tiered',
 401                'jitstressregs1',
 402                'jitstressregs2',
 403                'jitstressregs3',
 404                'jitstressregs4',
 405                'jitstressregs8',
 406                'jitstressregs0x10',
 407                'jitstressregs0x80',
 408                'jitstressregs0x1000',
 409                'jitstress2_jitstressregs1',
 410                'jitstress2_jitstressregs2',
 411                'jitstress2_jitstressregs3',
 412                'jitstress2_jitstressregs4',
 413                'jitstress2_jitstressregs8',
 414                'jitstress2_jitstressregs0x10',
 415                'jitstress2_jitstressregs0x80',
 416                'jitstress2_jitstressregs0x1000',
 417                'tailcallstress',
 418                // 'jitsse2only'                          // Only relevant to xarch
 419                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 420                // 'jitincompletehwintrinsic'
 421                // 'jitx86hwintrinsicnoavx'
 422                // 'jitx86hwintrinsicnoavx2'
 423                // 'jitx86hwintrinsicnosimd'
 424                // 'jitnox86hwintrinsic'
 425                'corefx_baseline',
 426                'corefx_minopts',
 427                'corefx_tieredcompilation',
 428                'corefx_jitstress1',
 429                'corefx_jitstress2',
 430                'corefx_jitstressregs1',
 431                'corefx_jitstressregs2',
 432                'corefx_jitstressregs3',
 433                'corefx_jitstressregs4',
 434                'corefx_jitstressregs8',
 435                'corefx_jitstressregs0x10',
 436                'corefx_jitstressregs0x80',
 437                'corefx_jitstressregs0x1000',
 438                'gcstress0x3',
 439                'gcstress0xc',
 440                'zapdisable',
 441                'heapverify1',
 442                'gcstress0xc_zapdisable',
 443                'gcstress0xc_zapdisable_jitstress2',
 444                'gcstress0xc_zapdisable_heapverify1',
 445                'gcstress0xc_jitstress1',
 446                'gcstress0xc_jitstress2',
 447                'gcstress0xc_minopts_heapverify1'
 448     ]
 449
 450     def static validLinuxArm64Scenarios = [
 451                'innerloop',
 452                'normal',
 453                // 'ilrt'
 454                'r2r',
 455                // 'longgc'
 456                // 'formatting'
 457                // 'gcsimulator'
 458                // 'jitdiff'
 459                // 'standalone_gc'
 460                // 'gc_reliability_framework'
 461                // 'illink'
 462                // 'corefx_innerloop'
 463                // 'crossgen_comparison'
 464                'r2r_jitstress1',
 465                'r2r_jitstress2',
 466                'r2r_jitstress1_tiered',
 467                'r2r_jitstress2_tiered',
 468                'r2r_jitstressregs1',
 469                'r2r_jitstressregs2',
 470                'r2r_jitstressregs3',
 471                'r2r_jitstressregs4',
 472                'r2r_jitstressregs8',
 473                'r2r_jitstressregs0x10',
 474                'r2r_jitstressregs0x80',
 475                'r2r_jitstressregs0x1000',
 476                'r2r_jitminopts',
 477                'r2r_jitforcerelocs',
 478                'r2r_gcstress15',
 479                'r2r_no_tiered_compilation',
 480                'minopts',
 481                'tieredcompilation',
 482                'no_tiered_compilation',
 483                'no_tiered_compilation_innerloop',
 484                'forcerelocs',
 485                'jitstress1',
 486                'jitstress2',
 487                'jitstress1_tiered',
 488                'jitstress2_tiered',
 489                'jitstressregs1',
 490                'jitstressregs2',
 491                'jitstressregs3',
 492                'jitstressregs4',
 493                'jitstressregs8',
 494                'jitstressregs0x10',
 495                'jitstressregs0x80',
 496                'jitstressregs0x1000',
 497                'jitstress2_jitstressregs1',
 498                'jitstress2_jitstressregs2',
 499                'jitstress2_jitstressregs3',
 500                'jitstress2_jitstressregs4',
 501                'jitstress2_jitstressregs8',
 502                'jitstress2_jitstressregs0x10',
 503                'jitstress2_jitstressregs0x80',
 504                'jitstress2_jitstressregs0x1000',
 505                'tailcallstress',
 506                // 'jitsse2only'                         // Only relevant to xarch
 507                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 508                // 'jitincompletehwintrinsic'
 509                // 'jitx86hwintrinsicnoavx'
 510                // 'jitx86hwintrinsicnoavx2'
 511                // 'jitx86hwintrinsicnosimd'
 512                // 'jitnox86hwintrinsic'
 513                'corefx_baseline',
 514                'corefx_minopts',
 515                'corefx_tieredcompilation',
 516                'corefx_jitstress1',
 517                'corefx_jitstress2',
 518                'corefx_jitstressregs1',
 519                'corefx_jitstressregs2',
 520                'corefx_jitstressregs3',
 521                'corefx_jitstressregs4',
 522                'corefx_jitstressregs8',
 523                'corefx_jitstressregs0x10',
 524                'corefx_jitstressregs0x80',
 525                'corefx_jitstressregs0x1000',
 526                'gcstress0x3',
 527                'gcstress0xc',
 528                'zapdisable',
 529                'heapverify1',
 530                'gcstress0xc_zapdisable',
 531                'gcstress0xc_zapdisable_jitstress2',
 532                'gcstress0xc_zapdisable_heapverify1',
 533                'gcstress0xc_jitstress1',
 534                'gcstress0xc_jitstress2',
 535                'gcstress0xc_minopts_heapverify1'
 536     ]
 537
 538     def static configurationList = ['Debug', 'Checked', 'Release']
 539
 540     // This is the set of architectures
 541     // Some of these are pseudo-architectures:
 542     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 543     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 544     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 545     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 546
 547     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 548     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 549 }
 550
 551 // **************************************************************
 552 // Create some specific views
 553 //
 554 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 555 // views based on a single regular expression. These views will be generated by adding a
 556 // specific set of jobs to them.
 557 //
 558 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 559 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 560 // those are required, don't add them (which simplifies the view pages, as well).
 561 // **************************************************************
 562
 563 class Views {
 564     def static MergeJobView = null
 565     def static PeriodicJobView = null
 566     def static ArchitectureViews = [:]
 567     def static OSViews = [:]
 568 }
 569
 570 // MergeJobView: include all jobs that execute when a PR change is merged.
 571 Views.MergeJobView = listView('Merge') {
 572     recurse()
 573     columns {
 574         status()
 575         weather()
 576         name()
 577         lastSuccess()
 578         lastFailure()
 579         lastDuration()
 580         buildButton()
 581     }
 582 }
 583
 584 // PeriodicJobView: include all jobs that execute on a schedule
 585 Views.PeriodicJobView = listView('Periodic') {
 586     recurse()
 587     columns {
 588         status()
 589         weather()
 590         name()
 591         lastSuccess()
 592         lastFailure()
 593         lastDuration()
 594         buildButton()
 595     }
 596 }
 597
 598 // Create a view for non-PR jobs for each architecture.
 599 Constants.architectureList.each { architecture ->
 600     Views.ArchitectureViews[architecture] = listView(architecture) {
 601         recurse()
 602         columns {
 603             status()
 604             weather()
 605             name()
 606             lastSuccess()
 607             lastFailure()
 608             lastDuration()
 609             buildButton()
 610         }
 611     }
 612 }
 613
 614 // Create a view for non-PR jobs for each OS.
 615 Constants.osList.each { os ->
 616     // Don't create one for the special 'Windows_NT_BuildOnly'
 617     if (os == 'Windows_NT_BuildOnly') {
 618         return
 619     }
 620     Views.OSViews[os] = listView(os) {
 621         recurse()
 622         columns {
 623             status()
 624             weather()
 625             name()
 626             lastSuccess()
 627             lastFailure()
 628             lastDuration()
 629             buildButton()
 630         }
 631     }
 632 }
 633
 634 def static addToMergeView(def job) {
 635     Views.MergeJobView.with {
 636         jobs {
 637             name(job.name)
 638         }
 639     }
 640 }
 641
 642 def static addToPeriodicView(def job) {
 643     Views.PeriodicJobView.with {
 644         jobs {
 645             name(job.name)
 646         }
 647     }
 648 }
 649
 650 def static addToViews(def job, def isPR, def architecture, def os) {
 651     if (isPR) {
 652         // No views want PR jobs currently.
 653         return
 654     }
 655
 656     // Add to architecture view.
 657     Views.ArchitectureViews[architecture].with {
 658         jobs {
 659             name(job.name)
 660         }
 661     }
 662
 663     // Add to OS view.
 664     Views.OSViews[os].with {
 665         jobs {
 666             name(job.name)
 667         }
 668     }
 669 }
 670
 671 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 672     addToPeriodicView(job)
 673     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 674 }
 675
 676 def static addGithubPushTriggerHelper(def job) {
 677     addToMergeView(job)
 678     Utilities.addGithubPushTrigger(job)
 679 }
 680
 681
 682 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 683     assert os instanceof String
 684     assert architecture instanceof String
 685
 686     def armArches = ['arm', 'armem', 'arm64']
 687
 688     if (!(architecture in armArches)) {
 689         assert options == null
 690         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 691
 692         return
 693     }
 694
 695     // This is an arm(64) job.
 696     //
 697     // There are several options.
 698     //
 699     // Windows_NT
 700     //
 701     // Arm32 (Build) -> latest-arm64
 702     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 703     // Arm32 (Test)  -> arm64-windows_nt
 704     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 705     //
 706     // Arm64 (Build) -> latest-arm64
 707     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 708     // Arm64 (Test)  -> arm64-windows_nt
 709     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 710     //
 711     // Ubuntu
 712     //
 713     // Arm32 emulator (Build, Test) -> arm-cross-latest
 714     //       |-> os == "Tizen" && (architecture == "armem")
 715     //
 716     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 717     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 718     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 719     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 720     // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
 721     //       |-> os == "Ubuntu" && (architecture == "arm")
 722     //
 723     // Arm64 (Build) -> arm64-cross-latest
 724     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
 725     // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
 726     //       |-> os != "Windows_NT" && architecture == "arm64"
 727     //
 728     // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
 729     // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
 730     //
 731     // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
 732     // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 733
 734     // This has to be a arm arch
 735     assert architecture in armArches
 736     if (os == "Windows_NT") {
 737         // arm32/arm64 Windows jobs share the same machines for now
 738         def isBuild = options['use_arm64_build_machine'] == true
 739
 740         if (isBuild == true) {
 741             job.with {
 742                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 743             }
 744         } else {
 745             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 746         }
 747     } else {
 748         assert os != 'Windows_NT'
 749
 750         if (architecture == 'armem') {
 751             // arm emulator (Tizen). Build and test on same machine,
 752             // using Docker.
 753             assert os == 'Tizen'
 754             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 755         }
 756         else {
 757             // arm/arm64 Ubuntu on hardware.
 758             assert architecture == 'arm' || architecture == 'arm64'
 759             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 760             def isBuild = (options != null) && (options['is_build_job'] == true)
 761             if (isFlow || isBuild) {
 762                 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 763                 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
 764                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 765             } else {
 766                 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
 767                 if (architecture == 'arm64') {
 768                     assert os == 'Ubuntu16.04'
 769                     job.with {
 770                         label('Ubuntu.1604.Arm64.Open')
 771                     }
 772                 }
 773                 else {
 774                     assert os == 'Ubuntu'
 775                     job.with {
 776                         label('ubuntu.1404.arm32.open')
 777                     }
 778                 }
 779             }
 780         }
 781     }
 782 }
 783
 784 // setJobMachineAffinity: compute the machine affinity options for a job,
 785 // then set the job with those affinity options.
 786 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 787 {
 788     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 789            (!isBuildJob && isTestJob  && !isFlowJob) ||
 790            (!isBuildJob && !isTestJob && isFlowJob)
 791
 792     def affinityOptions = null
 793     def affinityArchitecture = architecture
 794
 795     if (os == "Windows_NT") {
 796         if (architecture in Constants.armWindowsCrossArchitectureList) {
 797             if (isBuildJob) {
 798                 affinityOptions = [ "use_arm64_build_machine" : true ]
 799             } else if (isTestJob) {
 800                 affinityOptions = [ "use_arm64_build_machine" : false ]
 801             } else if (isFlowJob) {
 802                 // For the flow jobs set the machine affinity as x64
 803                 affinityArchitecture = 'x64'
 804             }
 805         }
 806     }
 807     else {
 808         if ((architecture == 'arm64') || (architecture == 'arm')) {
 809             if (isBuildJob) {
 810                 affinityOptions = ['is_build_job': true]
 811             } else if (isFlowJob) {
 812                 affinityOptions = ['is_flow_job': true]
 813             }
 814         }
 815     }
 816
 817     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 818 }
 819
 820 def static isGCStressRelatedTesting(def scenario) {
 821     // The 'r2r_gcstress15' scenario is a basic scenario.
 822     // Detect it and make it a GCStress related.
 823     if (scenario == 'r2r_gcstress15')
 824     {
 825         return true;
 826     }
 827
 828     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 829     def scenarioName = scenario.toLowerCase()
 830     def isGCStressTesting = false
 831     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 832         if (k in gcStressTestEnvVars) {
 833             isGCStressTesting = true;
 834         }
 835     }
 836     return isGCStressTesting
 837 }
 838
 839 def static isCoreFxScenario(def scenario) {
 840     def corefx_prefix = 'corefx_'
 841     if (scenario.length() < corefx_prefix.length()) {
 842         return false
 843     }
 844     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 845 }
 846
 847 def static isR2RBaselineScenario(def scenario) {
 848     return (scenario == 'r2r')
 849 }
 850
 851 def static isR2RStressScenario(def scenario) {
 852     return Constants.r2rStressScenarios.containsKey(scenario)
 853 }
 854
 855 def static isR2RScenario(def scenario) {
 856     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 857 }
 858
 859 def static isJitStressScenario(def scenario) {
 860     return Constants.jitStressModeScenarios.containsKey(scenario)
 861 }
 862
 863 def static isLongGc(def scenario) {
 864     return (scenario == 'longgc' || scenario == 'gcsimulator')
 865 }
 866
 867 def static isJitDiff(def scenario) {
 868     return (scenario == 'jitdiff')
 869 }
 870
 871 def static isGcReliabilityFramework(def scenario) {
 872     return (scenario == 'gc_reliability_framework')
 873 }
 874
 875 def static isArmWindowsScenario(def scenario) {
 876     return Constants.validArmWindowsScenarios.containsKey(scenario)
 877 }
 878
 879 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 880     if (isBuildOnly == true) {
 881         os = 'Windows_NT_BuildOnly'
 882     }
 883
 884     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 885     if (validOsPrTriggerArchConfigs != null) {
 886         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 887         if (validOsPrTriggerConfigs != null) {
 888             if (configuration in validOsPrTriggerConfigs) {
 889                 return true
 890             }
 891         }
 892     }
 893
 894     return false
 895 }
 896
 897 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 898 // scheduled with a default PR trigger despite the correlation being true at the moment.
 899 def static isInnerloopTestScenario(def scenario) {
 900     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 901 }
 902
 903 def static isCrossGenComparisonScenario(def scenario) {
 904     return (scenario == 'crossgen_comparison')
 905 }
 906
 907 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 908     assert isCrossGenComparisonScenario(scenario)
 909     return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
 910 }
 911
 912 def static getFxBranch(def branch) {
 913     def fxBranch = branch
 914     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 915     // branch even though CoreFX doesn't have such a branch.
 916     if (branch == 'dev/unix_test_workflow') {
 917         fxBranch = 'master'
 918     }
 919     return fxBranch
 920 }
 921
 922 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 923     // 2 hours (120 minutes) is the default timeout
 924     def timeout = 120
 925
 926     if (!isInnerloopTestScenario(scenario)) {
 927         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 928         timeout = 240
 929     }
 930
 931     if (!isBuildOnly) {
 932         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 933         if (isGCStressRelatedTesting(scenario)) {
 934             timeout = 4320
 935         }
 936         else if (isCoreFxScenario(scenario)) {
 937             timeout = 360
 938         }
 939         else if (isJitStressScenario(scenario)) {
 940             timeout = 300
 941         }
 942         else if (isR2RBaselineScenario(scenario)) {
 943             timeout = 240
 944         }
 945         else if (isLongGc(scenario)) {
 946             timeout = 1440
 947         }
 948         else if (isJitDiff(scenario)) {
 949             timeout = 240
 950         }
 951         else if (isGcReliabilityFramework(scenario)) {
 952             timeout = 1440
 953         }
 954         else if (architecture == 'armem' || architecture == 'arm64') {
 955             timeout = 240
 956         }
 957
 958         if (architecture == 'arm') {
 959             // ARM32 machines are particularly slow.
 960             timeout += 120
 961         }
 962     }
 963
 964     if (configuration == 'Debug') {
 965         // Debug runs can be very slow. Add an hour.
 966         timeout += 60
 967     }
 968
 969     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 970         // AltJit runs compile all methods twice.
 971         timeout *= 2
 972     }
 973
 974     // If we've changed the timeout from the default, set it in the job.
 975
 976     if (timeout != 120) {
 977         Utilities.setJobTimeout(newJob, timeout)
 978     }
 979 }
 980
 981 def static getJobFolder(def scenario) {
 982     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 983         return 'jitstress'
 984     }
 985     if (scenario == 'illink') {
 986         return 'illink'
 987     }
 988     return ''
 989 }
 990
 991 def static getStressModeDisplayName(def scenario) {
 992     def displayStr = ''
 993     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 994         def prefixLength = 'COMPlus_'.length()
 995         if (k.length() >= prefixLength) {
 996             def modeName = k.substring(prefixLength, k.length())
 997             displayStr += ' ' + modeName + '=' + v
 998         }
 999     }
1000
1001     if (isCoreFxScenario(scenario)) {
1002         displayStr = ('CoreFx ' + displayStr).trim()
1003     }
1004
1005     return displayStr
1006 }
1007
1008 def static getR2RDisplayName(def scenario) {
1009     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1010     def displayStr = scenario
1011     def prefixLength = 'r2r_'.length()
1012     if (displayStr.length() >= prefixLength) {
1013         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1014     } else if (scenario == 'r2r') {
1015         displayStr = "R2R"
1016     }
1017     return displayStr
1018 }
1019
1020 def static getScenarioDisplayString(def scenario) {
1021     switch (scenario) {
1022         case 'innerloop':
1023             return "Innerloop Build and Test"
1024
1025         case 'no_tiered_compilation_innerloop':
1026             def displayStr = getStressModeDisplayName(scenario)
1027             return "Innerloop Build and Test (Jit - ${displayStr})"
1028
1029         case 'corefx_innerloop':
1030             return "CoreFX Tests"
1031
1032         case 'normal':
1033             return "Build and Test"
1034
1035         case 'jitdiff':
1036             return "Jit Diff Build and Test"
1037
1038         case 'ilrt':
1039             return "IL RoundTrip Build and Test"
1040
1041         case 'longgc':
1042             return "Long-Running GC Build & Test"
1043
1044         case 'gcsimulator':
1045             return "GC Simulator"
1046
1047         case 'standalone_gc':
1048             return "Standalone GC"
1049
1050         case 'gc_reliability_framework':
1051             return "GC Reliability Framework"
1052
1053         case 'illink':
1054             return "via ILLink"
1055
1056         default:
1057             if (isJitStressScenario(scenario)) {
1058                 def displayStr = getStressModeDisplayName(scenario)
1059                 return "Build and Test (Jit - ${displayStr})"
1060             }
1061             else if (isR2RScenario(scenario)) {
1062                 def displayStr = getR2RDisplayName(scenario)
1063                 return "${displayStr} Build and Test"
1064             }
1065             else {
1066                 return "${scenario}"
1067             }
1068             break
1069     }
1070
1071     println("Unknown scenario: ${scenario}");
1072     assert false
1073 }
1074
1075 //
1076 // Functions to create an environment script.
1077 //      envScriptCreate -- initialize the script (call first)
1078 //      envScriptFinalize -- finalize the script (call last)
1079 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1080 //      envScriptAppendExistingScript -- append an existing script to the generated script
1081 //
1082 // Each script returns a string of commands. Concatenate all the strings together before
1083 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1084 //
1085
1086 // Initialize the environment setting script.
1087 def static envScriptCreate(def os, def stepScriptLocation) {
1088     def stepScript = ''
1089     if (os == 'Windows_NT') {
1090         stepScript += "echo Creating TestEnv script\r\n"
1091         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1092
1093         // Create at least an empty script.
1094         stepScript += "echo. > ${stepScriptLocation}\r\n"
1095     }
1096     else {
1097         stepScript += "echo Creating environment setting script\n"
1098         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1099     }
1100
1101     return stepScript
1102 }
1103
1104 // Generates the string for setting stress mode variables.
1105 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1106     def stepScript = ''
1107     if (os == 'Windows_NT') {
1108         stressModeVars.each{ k, v ->
1109             // Write out what we are writing to the script file
1110             stepScript += "echo Setting ${k}=${v}\r\n"
1111             // Write out the set itself to the script file`
1112             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1113         }
1114     }
1115     else {
1116         stressModeVars.each{ k, v ->
1117             // Write out what we are writing to the script file
1118             stepScript += "echo Setting ${k}=${v}\n"
1119             // Write out the set itself to the script file`
1120             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1121         }
1122     }
1123
1124     return stepScript
1125 }
1126
1127 // Append an existing script to an environment script.
1128 // Returns string of commands to do this.
1129 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1130     assert (os == 'Windows_NT')
1131     def stepScript = ''
1132
1133     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1134     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1135
1136     return stepScript
1137 }
1138
1139 // Finalize an environment setting script.
1140 // Returns string of commands to do this.
1141 def static envScriptFinalize(def os, def stepScriptLocation) {
1142     def stepScript = ''
1143
1144     if (os == 'Windows_NT') {
1145         // Display the resulting script. This is useful when looking at the output log file.
1146         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1147         stepScript += "type ${stepScriptLocation}\r\n"
1148     }
1149     else {
1150         stepScript += "chmod +x ${stepScriptLocation}\n"
1151     }
1152
1153     return stepScript
1154 }
1155
1156 def static isNeedDocker(def architecture, def os, def isBuild) {
1157     if (isBuild) {
1158         if (architecture == 'x86' && os == 'Ubuntu') {
1159             return true
1160         }
1161         else if (architecture == 'armem') {
1162             return true
1163         }
1164         else if (architecture == 'arm') {
1165             if (os == 'Ubuntu') {
1166                 return true
1167             }
1168         }
1169         else if (architecture == 'arm64') {
1170             if (os == 'Ubuntu16.04') {
1171                 return true
1172             }
1173         }
1174     }
1175     else {
1176         if (architecture == 'x86' && os == 'Ubuntu') {
1177             return true
1178         }
1179     }
1180     return false
1181 }
1182
1183 def static getDockerImageName(def architecture, def os, def isBuild) {
1184     // We must change some docker private images to official later
1185     if (isBuild) {
1186         if (architecture == 'x86' && os == 'Ubuntu') {
1187             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1188         }
1189         else if (architecture == 'armem') {
1190             if (os == 'Tizen') {
1191                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1192             }
1193         }
1194         else if (architecture == 'arm') {
1195             if (os == 'Ubuntu') {
1196                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1197             }
1198         }
1199         else if (architecture == 'arm64') {
1200             if (os == 'Ubuntu16.04') {
1201                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1202             }
1203         }
1204     }
1205     else {
1206         if (architecture == 'x86' && os == 'Ubuntu') {
1207             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1208         }
1209     }
1210     println("Unknown architecture to use docker: ${architecture} ${os}");
1211     assert false
1212 }
1213
1214
1215 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1216 // and only allowing using this hardware in some specific branches.
1217 def static jobRequiresLimitedHardware(def architecture, def os) {
1218     if (architecture == 'arm') {
1219         // arm Windows and Linux hardware is limited.
1220         return true
1221     }
1222     else if (architecture == 'arm64') {
1223         if (os == 'Windows_NT') {
1224             // arm64 Windows hardware is limited.
1225             return true
1226         }
1227         else {
1228             // arm64 Linux hardware is fast enough to allow more frequent jobs
1229             return false
1230         }
1231     }
1232     else {
1233         return false
1234     }
1235 }
1236
1237 // Calculates the name of the build job based on some typical parameters.
1238 //
1239 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1240     // If the architecture is x64, do not add that info into the build name.
1241     // Need to change around some systems and other builds to pick up the right builds
1242     // to do that.
1243
1244     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1245     if (isBuildOnly) {
1246         suffix += '_bld'
1247     }
1248     def baseName = ''
1249     switch (architecture) {
1250         case 'x64':
1251             if (scenario == 'normal') {
1252                 // For now we leave x64 off of the name for compatibility with other jobs
1253                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1254             }
1255             else if (scenario == 'formatting') {
1256                 // we don't care about the configuration for the formatting job. It runs all configs
1257                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1258             }
1259             else {
1260                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1261             }
1262             break
1263         case 'armem':
1264             // These are cross builds
1265             assert os == 'Tizen'
1266             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1267             break
1268         case 'arm':
1269         case 'arm64':
1270             // These are cross builds
1271             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1272             break
1273         case 'x86':
1274         case 'x86_arm_altjit':
1275         case 'x64_arm64_altjit':
1276             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1277             break
1278         default:
1279             println("Unknown architecture: ${architecture}");
1280             assert false
1281             break
1282     }
1283
1284     return baseName + suffix
1285 }
1286
1287 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1288     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1289
1290     // Limited hardware is restricted for non-PR triggers to certain branches.
1291     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1292         return
1293     }
1294
1295     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1296     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1297     if (architecture == 'x86' && os == 'Ubuntu') {
1298         return
1299     }
1300
1301     // Check scenario.
1302     switch (scenario) {
1303         case 'innerloop':
1304         case 'no_tiered_compilation_innerloop':
1305             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1306             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1307             // We should never get here (in the "innerloop cases) for non-PR jobs, except for this TEMPORARY exception.
1308             assert (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked'))
1309             addGithubPushTriggerHelper(job)
1310             break
1311
1312         case 'crossgen_comparison':
1313             break
1314         case 'normal':
1315             switch (architecture) {
1316                 case 'x64':
1317                 case 'x86':
1318                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1319                         addPeriodicTriggerHelper(job, '@daily')
1320                     }
1321                     else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1322                         addGithubPushTriggerHelper(job)
1323                     }
1324                     break
1325                 case 'arm64':
1326                     if (os == 'Windows_NT') {
1327                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1328                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1329                             // Do the builds daily.
1330                             addPeriodicTriggerHelper(job, '@daily')
1331                         }
1332                     }
1333                     else {
1334                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1335                         if (isFlowJob) {
1336                             addPeriodicTriggerHelper(job, '@daily')
1337                         }
1338                     }
1339                     break
1340                 case 'arm':
1341                     if (os == 'Windows_NT') {
1342                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1343                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1344                             // Do the builds daily.
1345                             addPeriodicTriggerHelper(job, '@daily')
1346                         }
1347                     }
1348                     else {
1349                         assert os == 'Ubuntu'
1350                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1351                         if (isFlowJob) {
1352                             // Currently no push triggers, with limited arm Linux hardware.
1353                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1354                             addPeriodicTriggerHelper(job, '@daily')
1355                         }
1356                     }
1357                     break
1358                 case 'armem':
1359                     addGithubPushTriggerHelper(job)
1360                     break
1361                 case 'x86_arm_altjit':
1362                 case 'x64_arm64_altjit':
1363                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1364                     if (configuration == 'Checked') {
1365                         addGithubPushTriggerHelper(job)
1366                     }
1367                     break
1368                 default:
1369                     println("Unknown architecture: ${architecture}");
1370                     assert false
1371                     break
1372             }
1373             break
1374         case 'r2r':
1375             assert !(os in bidailyCrossList)
1376             // r2r gets a push trigger for checked/release
1377             if (configuration == 'Checked' || configuration == 'Release') {
1378                 if (architecture == 'x64' && os != 'OSX10.12') {
1379                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1380                     if (isFlowJob || os == 'Windows_NT') {
1381                         addGithubPushTriggerHelper(job)
1382                     }
1383                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1384                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1385                     if (isFlowJob) {
1386                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1387                     }
1388                 }
1389                 // For x86, only add per-commit jobs for Windows
1390                 else if (architecture == 'x86') {
1391                     if (os == 'Windows_NT') {
1392                         addGithubPushTriggerHelper(job)
1393                     }
1394                 }
1395                 // arm r2r jobs should only run weekly.
1396                 else if (architecture == 'arm') {
1397                     if (isFlowJob) {
1398                         addPeriodicTriggerHelper(job, '@weekly')
1399                     }
1400                 }
1401                 // arm64 r2r jobs should only run weekly.
1402                 else if (architecture == 'arm64') {
1403                     if (isFlowJob) {
1404                         addPeriodicTriggerHelper(job, '@weekly')
1405                     }
1406                 }
1407             }
1408             break
1409         case 'r2r_jitstress1':
1410         case 'r2r_jitstress2':
1411         case 'r2r_jitstress1_tiered':
1412         case 'r2r_jitstress2_tiered':
1413         case 'r2r_jitstressregs1':
1414         case 'r2r_jitstressregs2':
1415         case 'r2r_jitstressregs3':
1416         case 'r2r_jitstressregs4':
1417         case 'r2r_jitstressregs8':
1418         case 'r2r_jitstressregs0x10':
1419         case 'r2r_jitstressregs0x80':
1420         case 'r2r_jitstressregs0x1000':
1421         case 'r2r_jitminopts':
1422         case 'r2r_jitforcerelocs':
1423         case 'r2r_gcstress15':
1424         case 'r2r_no_tiered_compilation':
1425             assert !(os in bidailyCrossList)
1426
1427             // GCStress=C is currently not supported on OS X
1428             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1429                 break
1430             }
1431
1432             if (configuration == 'Checked' || configuration == 'Release') {
1433                 if (architecture == 'x64') {
1434                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1435                     if (isFlowJob || os == 'Windows_NT') {
1436                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1437                     }
1438                 }
1439                 // For x86, only add periodic jobs for Windows
1440                 else if (architecture == 'x86') {
1441                     if (os == 'Windows_NT') {
1442                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1443                     }
1444                 }
1445                 else if (architecture == 'arm') {
1446                     if (isFlowJob) {
1447                         addPeriodicTriggerHelper(job, '@weekly')
1448                     }
1449                 }
1450                 else if (architecture == 'arm64') {
1451                     if (isFlowJob) {
1452                         addPeriodicTriggerHelper(job, '@weekly')
1453                     }
1454                 }
1455             }
1456             break
1457         case 'longgc':
1458             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1459             assert configuration == 'Release'
1460             assert architecture == 'x64'
1461             addPeriodicTriggerHelper(job, '@daily')
1462             // TODO: Add once external email sending is available again
1463             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1464             break
1465         case 'gcsimulator':
1466             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1467             assert configuration == 'Release'
1468             assert architecture == 'x64'
1469             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1470             // TODO: Add once external email sending is available again
1471             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1472             break
1473         case 'standalone_gc':
1474             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1475             assert (configuration == 'Release' || configuration == 'Checked')
1476             // TODO: Add once external email sending is available again
1477             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1478             addPeriodicTriggerHelper(job, '@daily')
1479             break
1480         case 'gc_reliability_framework':
1481             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1482             assert (configuration == 'Release' || configuration == 'Checked')
1483             // Only triggered by phrase.
1484             break
1485         case 'ilrt':
1486             assert !(os in bidailyCrossList)
1487             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1488             if (architecture == 'x64' && configuration == 'Release') {
1489                 if (isFlowJob || os == 'Windows_NT') {
1490                     addPeriodicTriggerHelper(job, '@daily')
1491                 }
1492             }
1493             break
1494         case 'jitdiff':
1495             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1496             assert configuration == 'Checked'
1497             assert (architecture == 'x64' || architecture == 'x86')
1498             addGithubPushTriggerHelper(job)
1499             break
1500         case 'formatting':
1501             assert (os == 'Windows_NT' || os == "Ubuntu")
1502             assert architecture == 'x64'
1503             addGithubPushTriggerHelper(job)
1504             break
1505         case 'jitstressregs1':
1506         case 'jitstressregs2':
1507         case 'jitstressregs3':
1508         case 'jitstressregs4':
1509         case 'jitstressregs8':
1510         case 'jitstressregs0x10':
1511         case 'jitstressregs0x80':
1512         case 'jitstressregs0x1000':
1513         case 'minopts':
1514         case 'tieredcompilation':
1515         case 'no_tiered_compilation':
1516         case 'forcerelocs':
1517         case 'jitstress1':
1518         case 'jitstress2':
1519         case 'jitstress1_tiered':
1520         case 'jitstress2_tiered':
1521         case 'jitstress2_jitstressregs1':
1522         case 'jitstress2_jitstressregs2':
1523         case 'jitstress2_jitstressregs3':
1524         case 'jitstress2_jitstressregs4':
1525         case 'jitstress2_jitstressregs8':
1526         case 'jitstress2_jitstressregs0x10':
1527         case 'jitstress2_jitstressregs0x80':
1528         case 'jitstress2_jitstressregs0x1000':
1529         case 'tailcallstress':
1530         case 'jitsse2only':
1531         case 'jitnosimd':
1532         case 'jitnox86hwintrinsic':
1533         case 'jitincompletehwintrinsic':
1534         case 'jitx86hwintrinsicnoavx':
1535         case 'jitx86hwintrinsicnoavx2':
1536         case 'jitx86hwintrinsicnosimd':
1537         case 'corefx_baseline':
1538         case 'corefx_minopts':
1539         case 'corefx_tieredcompilation':
1540         case 'corefx_jitstress1':
1541         case 'corefx_jitstress2':
1542         case 'corefx_jitstressregs1':
1543         case 'corefx_jitstressregs2':
1544         case 'corefx_jitstressregs3':
1545         case 'corefx_jitstressregs4':
1546         case 'corefx_jitstressregs8':
1547         case 'corefx_jitstressregs0x10':
1548         case 'corefx_jitstressregs0x80':
1549         case 'corefx_jitstressregs0x1000':
1550         case 'zapdisable':
1551             if (os == 'CentOS7.1') {
1552                 break
1553             }
1554             if (os in bidailyCrossList) {
1555                 break
1556             }
1557             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1558             // build for the flow job. We don't need cron jobs for these. Note that the
1559             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1560             // function very early, so only non-Windows gets here.
1561             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1562                 break
1563             }
1564             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1565                 break
1566             }
1567             if (jobRequiresLimitedHardware(architecture, os)) {
1568                 addPeriodicTriggerHelper(job, '@weekly')
1569             }
1570             else {
1571                 addPeriodicTriggerHelper(job, '@daily')
1572             }
1573             break
1574         case 'heapverify1':
1575         case 'gcstress0x3':
1576             if (os == 'CentOS7.1') {
1577                 break
1578             }
1579             if (os in bidailyCrossList) {
1580                 break
1581             }
1582             addPeriodicTriggerHelper(job, '@weekly')
1583             break
1584         case 'gcstress0xc':
1585         case 'gcstress0xc_zapdisable':
1586         case 'gcstress0xc_zapdisable_jitstress2':
1587         case 'gcstress0xc_zapdisable_heapverify1':
1588         case 'gcstress0xc_jitstress1':
1589         case 'gcstress0xc_jitstress2':
1590         case 'gcstress0xc_minopts_heapverify1':
1591             if (os == 'OSX10.12') {
1592                 // GCStress=C is currently not supported on OS X
1593                 break
1594             }
1595             if (os == 'CentOS7.1') {
1596                 break
1597             }
1598             if (os in bidailyCrossList) {
1599                 break
1600             }
1601             addPeriodicTriggerHelper(job, '@weekly')
1602             break
1603
1604         case 'illink':
1605             // Testing on other operating systems TBD
1606             assert (os == 'Windows_NT' || os == 'Ubuntu')
1607             if (architecture == 'x64' || architecture == 'x86') {
1608                 if (configuration == 'Checked') {
1609                     addPeriodicTriggerHelper(job, '@daily')
1610                 }
1611             }
1612             break
1613
1614         default:
1615             println("Unknown scenario: ${scenario}");
1616             assert false
1617             break
1618     }
1619     return
1620 }
1621
1622 // **************************
1623 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1624 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1625 // builds will do a couple extra steps.
1626 // **************************
1627
1628 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1629 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1630 // it's just used for internal builds
1631 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1632 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1633     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1634
1635     if (isWindowsBuildOnlyJob) {
1636         return
1637     }
1638
1639     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1640     // Non pull request builds.
1641     if (!isPR) {
1642         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1643         return
1644     }
1645
1646     def arm64Users = [
1647         'adityamandaleeka',
1648         'AndyAyersMS',
1649         'briansull',
1650         'BruceForstall',
1651         'CarolEidt',
1652         'davidwrighton',
1653         'echesakovMSFT',
1654         'erozenfeld',
1655         'janvorli',
1656         'jashook',
1657         'pgodeq',
1658         'RussKeldorph',
1659         'sandreenko',
1660         'sdmaclea',
1661         'swaroop-sridhar',
1662         'jkotas',
1663         'markwilkie',
1664         'weshaggard'
1665     ]
1666
1667     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1668     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1669
1670     def contextString = ""
1671     def triggerString = ""
1672     def needsTrigger = true
1673     def isDefaultTrigger = false
1674     def isArm64PrivateJob = false
1675     def scenarioString = ""
1676
1677     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1678     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1679     // some jobs where the Debug configuration just does builds, no tests.
1680     //
1681     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1682     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1683     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1684     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1685
1686     if (architecture == 'armem') {
1687         assert os == 'Tizen'
1688         architecture = 'armel'
1689     }
1690
1691     switch (architecture) {
1692         case 'x64_arm64_altjit':
1693         case 'x86_arm_altjit':
1694             // TODO: for consistency, add "Build and Test" at end.
1695             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1696             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1697             break
1698
1699         case 'armel':
1700         case 'arm':
1701         case 'arm64':
1702             contextString = "${os} ${architecture} Cross ${configuration}"
1703             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1704
1705             if (scenario == 'innerloop') {
1706                 contextString += " Innerloop"
1707                 triggerString += "\\W+Innerloop"
1708             }
1709             else {
1710                 contextString += " ${scenario}"
1711                 triggerString += "\\W+${scenario}"
1712             }
1713
1714             if (configuration == 'Debug') {
1715                 contextString += " Build"
1716                 triggerString += "\\W+Build"
1717             }
1718             else {
1719                 contextString += " Build and Test"
1720                 triggerString += "\\W+Build and Test"
1721             }
1722
1723             triggerString += ".*"
1724             break
1725
1726         default:
1727             scenarioString = getScenarioDisplayString(scenario)
1728             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1729             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1730
1731             switch (scenario) {
1732                 case 'normal':
1733                     triggerString += "\\W+Build and Test.*"
1734                     break
1735
1736                 case 'corefx_innerloop': // maintain this asymmetry
1737                     triggerString += "\\W+CoreFX Tests.*"
1738                     break
1739
1740                 default:
1741                     triggerString += "\\W+${scenario}.*"
1742                     break
1743             }
1744
1745             triggerString += ".*"
1746             break
1747     }
1748
1749     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1750     // non-flow jobs that are only used as part of flow jobs.
1751
1752     switch (architecture) {
1753         case 'x64': // editor brace matching: {
1754             if (scenario == 'formatting') {
1755                 assert configuration == 'Checked'
1756                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1757                     isDefaultTrigger = true
1758                     contextString = "${os} ${architecture} Formatting"
1759                 }
1760                 break
1761             }
1762
1763             switch (os) {
1764                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1765                 case 'Debian8.4':
1766                 case 'RHEL7.2':
1767                     if (scenario == 'innerloop') {
1768                         assert !isFlowJob
1769                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1770                         isDefaultTrigger = true
1771                         break
1772                     }
1773
1774                     // fall through
1775
1776                 case 'Fedora24':
1777                 case 'Ubuntu16.04':
1778                 case 'Ubuntu16.10':
1779                     assert !isFlowJob
1780                     assert scenario != 'innerloop'
1781                     contextString = "${os} ${architecture} ${configuration} Build"
1782                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1783                     break
1784
1785                 case 'Ubuntu':
1786                     if (scenario == 'illink') {
1787                         break
1788                     }
1789                     else if (scenario == 'corefx_innerloop') {
1790                         if (configuration == 'Checked') {
1791                             isDefaultTrigger = true
1792                         }
1793                         break
1794                     }
1795
1796                     // fall through
1797
1798                 case 'OSX10.12':
1799                     // Triggers on the non-flow jobs aren't necessary here
1800                     // Corefx testing uses non-flow jobs.
1801                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1802                         needsTrigger = false
1803                         break
1804                     }
1805                     switch (scenario) {
1806                         case 'innerloop':
1807                             isDefaultTrigger = true
1808                             break
1809
1810                         case 'no_tiered_compilation_innerloop':
1811                             if (os == 'Ubuntu') {
1812                                 isDefaultTrigger = true
1813                             }
1814                             break
1815
1816                         default:
1817                             break
1818                     }
1819                     break
1820
1821                 case 'CentOS7.1':
1822                     switch (scenario) {
1823                         case 'innerloop':
1824                             // CentOS uses checked for default PR tests while debug is build only
1825                             if (configuration == 'Debug') {
1826                                 isDefaultTrigger = true
1827                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1828                                 break
1829                             }
1830
1831                             // Make sure this is a flow job to get build and test.
1832                             if (!isFlowJob) {
1833                                 needsTrigger = false
1834                                 break
1835                             }
1836
1837                             if (configuration == 'Checked') {
1838                                 assert job.name.contains("flow")
1839                                 isDefaultTrigger = true
1840                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1841                             }
1842                             break
1843
1844                         case 'normal':
1845                             // Make sure this is a flow job to get build and test.
1846                             if (!isFlowJob) {
1847                                 needsTrigger = false
1848                                 break
1849                             }
1850                             break
1851
1852                         default:
1853                             break
1854                     }
1855                     break
1856
1857                 case 'Windows_NT':
1858                     switch (scenario) {
1859                         case 'innerloop':
1860                         case 'no_tiered_compilation_innerloop':
1861                             isDefaultTrigger = true
1862                             break
1863
1864                         case 'corefx_innerloop':
1865                             if (configuration == 'Checked' || configuration == 'Release') {
1866                                 isDefaultTrigger = true
1867                             }
1868                             break
1869
1870                         default:
1871                             break
1872                     }
1873                     break
1874
1875                 default:
1876                     println("Unknown os: ${os}");
1877                     assert false
1878                     break
1879
1880             } // switch (os)
1881
1882             break
1883         // editor brace matching: }
1884
1885         case 'armel': // editor brace matching: {
1886             job.with {
1887                 publishers {
1888                     azureVMAgentPostBuildAction {
1889                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1890                     }
1891                 }
1892             }
1893
1894             switch (os) {
1895                 case 'Tizen':
1896                     if (scenario == 'innerloop') {
1897                         if (configuration == 'Checked') {
1898                             isDefaultTrigger = true
1899                         }
1900                     }
1901                     break
1902             }
1903
1904             break
1905         // editor brace matching: }
1906
1907         case 'arm':
1908         case 'arm64': // editor brace matching: {
1909
1910             switch (os) {
1911                 case 'Ubuntu':
1912                 case 'Ubuntu16.04':
1913
1914                     // Triggers on the non-flow jobs aren't necessary
1915                     if (!isFlowJob) {
1916                         needsTrigger = false
1917                         break
1918                     }
1919
1920                     switch (scenario) {
1921                         case 'innerloop':
1922                         case 'no_tiered_compilation_innerloop':
1923                             if (configuration == 'Checked') {
1924                                 // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1925                                 //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1926                                 if (architecture == 'arm64') {
1927                                     break
1928                                 }
1929                                 isDefaultTrigger = true
1930                             }
1931                             break
1932                     }
1933                     break
1934
1935                 case 'Windows_NT':
1936                     assert isArmWindowsScenario(scenario)
1937
1938                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1939                     // the non-flow Build job. All others need a trigger on the flow job.
1940                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1941                     if (isFlowJob != needsFlowJobTrigger) {
1942                         needsTrigger = false
1943                         break
1944                     }
1945
1946                     switch (scenario) {
1947                         case 'innerloop':
1948                             if (configuration == 'Debug') {
1949                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1950                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1951                                 // People in arm64Users will get both this and the Checked Build and Test job.
1952                                 isDefaultTrigger = true
1953                             } else if (configuration == 'Checked') {
1954                                 isDefaultTrigger = true
1955                                 isArm64PrivateJob = true
1956                             }
1957                             break
1958                         default:
1959                             isArm64PrivateJob = true
1960                             break
1961                     }
1962                     break
1963                 default:
1964                     println("NYI os: ${os}");
1965                     assert false
1966                     break
1967             }
1968             break
1969
1970         // editor brace matching: }
1971         case 'x86': // editor brace matching: {
1972             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1973             if (os == 'Ubuntu') {
1974                 // Triggers on the non-flow jobs aren't necessary here
1975                 if (!isFlowJob) {
1976                     needsTrigger = false
1977                     break
1978                 }
1979
1980                 // on-demand only for ubuntu x86
1981                 contextString = "${os} ${architecture} ${configuration} Build"
1982                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
1983                 break
1984             }
1985             switch (scenario) {
1986                 case 'innerloop':
1987                 case 'no_tiered_compilation_innerloop':
1988                     isDefaultTrigger = true
1989                     break
1990                 default:
1991                     break
1992             }
1993             break
1994
1995         // editor brace matching: }
1996         case 'x64_arm64_altjit':
1997         case 'x86_arm_altjit':
1998             // Everything default
1999             break
2000
2001         default:
2002             println("Unknown architecture: ${architecture}");
2003             assert false
2004             break
2005     }
2006
2007     if (needsTrigger) {
2008         if (isArm64PrivateJob) {
2009             if (isDefaultTrigger) {
2010                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2011             }
2012             else {
2013                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2014             }
2015         }
2016         else {
2017             if (isDefaultTrigger) {
2018                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2019             }
2020             else {
2021                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2022             }
2023         }
2024     }
2025 }
2026
2027 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2028     def buildCommands = []
2029     def osGroup = getOSGroup(os)
2030     def lowerConfiguration = configuration.toLowerCase()
2031
2032     def priority = '1'
2033     if (isInnerloopTestScenario(scenario)) {
2034         priority = '0'
2035     }
2036
2037     def doCoreFxTesting = isCoreFxScenario(scenario)
2038
2039     // Calculate the build steps, archival, and xunit results
2040     switch (os) {
2041         case 'Windows_NT': // editor brace matching: {
2042             switch (architecture) {
2043                 case 'x64':
2044                 case 'x86':
2045                 case 'x86_arm_altjit':
2046                 case 'x64_arm64_altjit':
2047                     def arch = architecture
2048                     def buildOpts = ''
2049                     if (architecture == 'x86_arm_altjit') {
2050                         arch = 'x86'
2051                     }
2052                     else if (architecture == 'x64_arm64_altjit') {
2053                         arch = 'x64'
2054                     }
2055
2056                     if (scenario == 'formatting') {
2057                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2058                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2059                         break
2060                     }
2061
2062                     if (scenario == 'illink') {
2063                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2064                     }
2065
2066                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2067                     if ((lowerConfiguration == 'release') &&
2068                         (scenario in Constants.basicScenarios) &&
2069                         (architecture != 'x86_arm_altjit') &&
2070                         (architecture != 'x64_arm64_altjit')) {
2071
2072                         buildOpts += ' -enforcepgo'
2073                     }
2074
2075                     if (doCoreFxTesting) {
2076                         buildOpts += ' skiptests';
2077                     } else {
2078                         buildOpts += " -priority=${priority}"
2079                     }
2080
2081                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2082                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2083                     // 35 characters long.
2084
2085                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2086
2087                     if (!isBuildOnly) {
2088                         def runtestArguments = ''
2089                         def testOpts = 'collectdumps'
2090
2091                         if (isR2RScenario(scenario)) {
2092
2093                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2094                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2095                             // to cause the tests to be crossgen'ed.
2096
2097                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2098                                 testOpts += ' crossgenaltjit protononjit.dll'
2099                             } else {
2100                                 testOpts += ' crossgen'
2101                             }
2102
2103                             testOpts += ' runcrossgentests'
2104                         }
2105                         else if (scenario == 'jitdiff') {
2106                             testOpts += ' jitdisasm crossgen'
2107                         }
2108                         else if (scenario == 'ilrt') {
2109                             testOpts += ' ilasmroundtrip'
2110                         }
2111                         else if (isLongGc(scenario)) {
2112                             testOpts += " ${scenario} sequential"
2113                         }
2114                         else if (scenario == 'standalone_gc') {
2115                             testOpts += ' gcname clrgc.dll'
2116                         }
2117                         else if (scenario == 'illink') {
2118                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2119                         }
2120
2121                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2122                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2123                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2124                         // which is read by the xunit harness.)
2125                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2126                         {
2127                             def timeout = 1800000
2128                             testOpts += " timeout ${timeout}"
2129                         }
2130
2131                         // If we are running a stress mode, we should write out the set of key
2132                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2133
2134                         def envScriptPath = ''
2135                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2136                             def buildCommandsStr = ''
2137                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2138                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2139
2140                             if (isJitStressScenario(scenario)) {
2141                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2142                             }
2143                             else if (isR2RStressScenario(scenario)) {
2144                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2145                             }
2146
2147                             if (architecture == 'x86_arm_altjit') {
2148                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2149                             }
2150                             else if (architecture == 'x64_arm64_altjit') {
2151                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2152                             }
2153
2154                             envScriptFinalize(os, envScriptPath)
2155
2156                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2157                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2158                             // into a single script.
2159                             buildCommands += buildCommandsStr
2160                         }
2161                         else if (architecture == 'x86_arm_altjit') {
2162                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2163                         }
2164                         else if (architecture == 'x64_arm64_altjit') {
2165                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2166                         }
2167                         if (envScriptPath != '') {
2168                             testOpts += " TestEnv ${envScriptPath}"
2169                         }
2170
2171                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2172
2173                         if (doCoreFxTesting) {
2174                             if (scenario == 'corefx_innerloop') {
2175                                 // Create CORE_ROOT and testhost
2176                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2177                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2178
2179                                 // Archive and process (only) the test results
2180                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2181                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2182                             }
2183                             else {
2184                                 def workspaceRelativeFxRoot = "_/fx"
2185                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2186                                 def fxBranch = getFxBranch(branch)
2187
2188                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2189
2190                                 // Archive and process (only) the test results
2191                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2192                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2193
2194                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2195                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2196                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2197                             }
2198                         }
2199                         else if (isGcReliabilityFramework(scenario)) {
2200                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2201                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2202                         }
2203                         else {
2204                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2205                         }
2206                     } // end if (!isBuildOnly)
2207
2208                     if (!doCoreFxTesting) {
2209                         // Run the rest of the build
2210                         // Build the mscorlib for the other OS's
2211                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2212                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2213
2214                         if (arch == 'x64') {
2215                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2216                         }
2217
2218                         // Zip up the tests directory so that we don't use so much space/time copying
2219                         // 10s of thousands of files around.
2220                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2221
2222                         if (!isJitStressScenario(scenario)) {
2223                             // For Windows, pull full test results and test drops for x86/x64.
2224                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2225                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2226                         }
2227
2228                         if (scenario == 'jitdiff') {
2229                             // retrieve jit-dasm output for base commit, and run jit-diff
2230                             if (!isBuildOnly) {
2231                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2232                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2233                             }
2234                         }
2235
2236                         if (!isBuildOnly) {
2237                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2238                         }
2239                     }
2240                     break
2241                 case 'arm':
2242                 case 'arm64':
2243                     assert isArmWindowsScenario(scenario)
2244
2245                     def buildOpts = ''
2246
2247                     if (doCoreFxTesting) {
2248                         buildOpts += ' skiptests'
2249                     } else {
2250                         buildOpts += " -priority=${priority}"
2251                     }
2252
2253                     // This is now a build only job. Do not run tests. Use the flow job.
2254                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2255
2256                     if (doCoreFxTesting) {
2257                         assert isBuildOnly
2258
2259                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2260                         def envScriptPath = ''
2261                         def buildCommandsStr = ''
2262                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2263                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2264                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2265                         envScriptFinalize(os, envScriptPath)
2266                         buildCommands += buildCommandsStr
2267
2268                         def workspaceRelativeFxRootLinux = "_/fx"
2269                         def workspaceRelativeFxRootWin = "_\\fx"
2270                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2271                         def fxBranch = getFxBranch(branch)
2272
2273                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2274
2275                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2276                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2277                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2278
2279                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2280                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2281                     } else {
2282                         // Zip up the tests directory so that we don't use so much space/time copying
2283                         // 10s of thousands of files around.
2284                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2285
2286                         // Add archival.
2287                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2288                     }
2289                     break
2290                 default:
2291                     println("Unknown architecture: ${architecture}");
2292                     assert false
2293                     break
2294             }
2295             break
2296         // end case 'Windows_NT'; editor brace matching: }
2297         case 'Ubuntu':
2298         case 'Ubuntu16.04':
2299         case 'Ubuntu16.10':
2300         case 'Debian8.4':
2301         case 'OSX10.12':
2302         case 'CentOS7.1':
2303         case 'RHEL7.2':
2304         case 'Tizen':
2305         case 'Fedora24': // editor brace matching: {
2306             switch (architecture) {
2307                 case 'x64':
2308                 case 'x86':
2309                     if (architecture == 'x86' && os == 'Ubuntu') {
2310                         // build and PAL test
2311                         def dockerImage = getDockerImageName(architecture, os, true)
2312                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2313                         dockerImage = getDockerImageName(architecture, os, false)
2314                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2315                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2316                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2317                         break
2318                     }
2319
2320                     if (scenario == 'formatting') {
2321                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2322                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2323                         break
2324                     }
2325
2326                     if (scenario == 'illink') {
2327                         assert(os == 'Ubuntu')
2328                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2329                     }
2330
2331                     if (!doCoreFxTesting) {
2332                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2333                         // only on supported OS platforms.
2334                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2335                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2336
2337                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2338                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2339
2340                         // Basic archiving of the build
2341                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2342                         // And pal tests
2343                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2344                     }
2345                     else {
2346                         if (scenario == 'corefx_innerloop') {
2347                             assert os == 'Ubuntu' || 'OSX10.12'
2348                             assert architecture == 'x64'
2349
2350                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2351                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2352                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2353
2354                             break
2355                             // Archive and process (only) the test results
2356                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2357                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2358                         }
2359                         else {
2360                             // Corefx stress testing
2361                             assert os == 'Ubuntu'
2362                             assert architecture == 'x64'
2363                             assert lowerConfiguration == 'checked'
2364                             assert isJitStressScenario(scenario)
2365
2366                             // Build coreclr
2367                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2368
2369                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2370
2371                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2372                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2373                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2374                             buildCommands += envScriptCmds
2375
2376                             // Build and text corefx
2377                             def workspaceRelativeFxRoot = "_/fx"
2378                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2379                             def fxBranch = getFxBranch(branch)
2380
2381                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2382
2383                             // Archive and process (only) the test results
2384                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2385                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2386                         }
2387                     }
2388                     break
2389                 case 'armem':
2390                     // Emulator cross builds for ARM runs on Tizen currently
2391                     assert os == 'Tizen'
2392
2393                     def arm_abi = "armel"
2394                     def linuxCodeName = "tizen"
2395
2396                     // Unzip the Windows test binaries first. Exit with 0
2397                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2398
2399                     // Unpack the corefx binaries
2400                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2401                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2402
2403                     // Call the ARM CI script to cross build and test using docker
2404                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2405                     --mode=docker \\
2406                     --${arm_abi} \\
2407                     --linuxCodeName=${linuxCodeName} \\
2408                     --buildConfig=${lowerConfiguration} \\
2409                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2410                     --coreFxBinDir=./bin/CoreFxBinDir \\
2411                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2412
2413                     // Basic archiving of the build, no pal tests
2414                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2415                     break
2416                 case 'arm64':
2417                 case 'arm':
2418                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2419                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2420
2421                     // Add some useful information to the log file. Ignore return codes.
2422                     buildCommands += "uname -a || true"
2423
2424                     def additionalOpts = ""
2425                     if (architecture == 'arm') {
2426                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2427                     }
2428
2429                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2430                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2431                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2432                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2433                     // ZIP up the generated CoreFX runtime and tests.
2434
2435                     def dockerImage = getDockerImageName(architecture, os, true)
2436                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2437
2438                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2439
2440                     if (doCoreFxTesting) {
2441                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2442
2443                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2444                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2445                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2446                         buildCommands += envScriptCmds
2447
2448                         // Build and text corefx
2449                         def workspaceRelativeFxRootLinux = "_/fx"
2450                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2451                         def fxBranch = getFxBranch(branch)
2452
2453                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2454
2455                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2456                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2457                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2458
2459                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2460                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2461                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2462                     }
2463                     else if (isCrossGenComparisonScenario(scenario)) {
2464                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2465
2466                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2467                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2468                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2469                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2470                         def workspaceRelativeResultsDir = "_"
2471                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2472                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2473                         getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
2474                             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2475                             def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2476
2477                             buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2478                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2479                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2480                         } // crossArch
2481                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2482                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2483                     }
2484                     else if (architecture == 'arm') {
2485                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2486                         // download the appropriate CoreFX packages.
2487                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2488                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2489
2490                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2491
2492                         // ZIP up for the test job (created in the flow job code):
2493                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2494                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2495                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2496                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2497
2498                         // These commands are assumed to be run from the root of the workspace.
2499                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2500                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2501
2502                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2503                     }
2504                     else {
2505                         assert architecture == 'arm64'
2506
2507                         // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2508                         // Linux/arm64 does not use Windows-built tests.
2509
2510                         def testBuildOpts = ""
2511                         if (priority == '1') {
2512                             testBuildOpts = "priority1"
2513                         }
2514
2515                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2516
2517                         // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2518                         buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
2519
2520                         // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
2521                         // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
2522                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2523
2524                         Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2525                     }
2526
2527                     // Archive the build logs from both product and test builds.
2528                     Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
2529
2530                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2531                     // the cleanup task in Jenkins can't remove.
2532                     newJob.with {
2533                         publishers {
2534                             azureVMAgentPostBuildAction {
2535                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2536                             }
2537                         }
2538                     }
2539                     break
2540                 default:
2541                     println("Unknown architecture: ${architecture}");
2542                     assert false
2543                     break
2544             }
2545             break
2546         // editor brace matching: }
2547         default:
2548             println("Unknown os: ${os}");
2549             assert false
2550             break
2551     } // os
2552
2553     return buildCommands
2554 }
2555
2556 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2557 // Returns true if the job should be generated.
2558 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2559 {
2560     // The various "innerloop" jobs are only available as PR triggered.
2561
2562     if (!isPR) {
2563         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
2564             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
2565             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
2566             return true
2567         }
2568
2569         if (isInnerloopTestScenario(scenario)) {
2570             return false
2571         }
2572
2573         if (scenario == 'corefx_innerloop') {
2574             return false
2575         }
2576     }
2577
2578     // Tizen is only supported for armem architecture
2579     if (os == 'Tizen' && architecture != 'armem') {
2580         return false
2581     }
2582
2583     // Filter based on architecture.
2584
2585     switch (architecture) {
2586         case 'arm':
2587             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2588                 return false
2589             }
2590             break
2591         case 'arm64':
2592             if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2593                 return false
2594             }
2595             break
2596         case 'armem':
2597             if (os != 'Tizen') {
2598                 return false
2599             }
2600             break
2601         case 'x86_arm_altjit':
2602         case 'x64_arm64_altjit':
2603             if (os != 'Windows_NT') {
2604                 return false
2605             }
2606             break
2607         case 'x86':
2608             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2609                 return false
2610             }
2611             break
2612         case 'x64':
2613             // Everything implemented
2614             break
2615         default:
2616             println("Unknown architecture: ${architecture}")
2617             assert false
2618             break
2619     }
2620
2621     // Which (Windows) build only jobs are required?
2622
2623     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2624
2625     if (isBuildOnly) {
2626         switch (architecture) {
2627             case 'arm':
2628             case 'arm64':
2629                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2630                 if (!isCoreFxScenario(scenario)) {
2631                     return false
2632                 }
2633                 break
2634             case 'x64':
2635             case 'x86':
2636                 if (!isNormalOrInnerloop) {
2637                     return false
2638                 }
2639                 break
2640             default:
2641                 return false
2642         }
2643     }
2644
2645     // Filter based on scenario.
2646
2647     if (isJitStressScenario(scenario)) {
2648         if (configuration != 'Checked') {
2649             return false
2650         }
2651
2652         def isEnabledOS = (os == 'Windows_NT') ||
2653                           (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2654                           (os == 'Ubuntu' && architecture == 'arm') ||
2655                           (os == 'Ubuntu16.04' && architecture == 'arm64')
2656         if (!isEnabledOS) {
2657             return false
2658         }
2659
2660         switch (architecture) {
2661             case 'x64':
2662             case 'x86_arm_altjit':
2663             case 'x64_arm64_altjit':
2664                 break
2665
2666             case 'x86':
2667                 // x86 ubuntu: no stress modes
2668                 if (os == 'Ubuntu') {
2669                     return false
2670                 }
2671                 break
2672
2673             case 'arm':
2674             case 'arm64':
2675                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2676                 // No "regular" Windows arm corefx jobs, e.g.
2677                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2678                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2679                 if (os == 'Windows_NT') {
2680                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2681                         return false
2682                     }
2683                 }
2684                 else {
2685                     if (!isCoreFxScenario(scenario)) {
2686                         return false
2687                     }
2688                 }
2689                 break
2690
2691             default:
2692                 // armem: no stress jobs for ARM emulator.
2693                 return false
2694         }
2695     }
2696     else if (isR2RScenario(scenario)) {
2697         if (os != 'Windows_NT') {
2698             return false
2699         }
2700
2701         if (isR2RBaselineScenario(scenario)) {
2702             // no need for Debug scenario; Checked is sufficient
2703             if (configuration != 'Checked' && configuration != 'Release') {
2704                 return false
2705             }
2706         }
2707         else if (isR2RStressScenario(scenario)) {
2708             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2709             if (configuration != 'Checked') {
2710                 return false
2711             }
2712         }
2713
2714         switch (architecture) {
2715             case 'arm':
2716             case 'arm64':
2717                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2718                 return false
2719
2720             default:
2721                 break
2722         }
2723     }
2724     else if (isCrossGenComparisonScenario(scenario)) {
2725         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2726     }
2727     else {
2728         // Skip scenarios
2729         switch (scenario) {
2730             case 'ilrt':
2731                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2732                 // the default scenario build
2733                 if (os != 'Windows_NT') {
2734                     return false
2735                 }
2736                 // Only x64 for now
2737                 if (architecture != 'x64') {
2738                     return false
2739                 }
2740                 // Release only
2741                 if (configuration != 'Release') {
2742                     return false
2743                 }
2744                 break
2745             case 'jitdiff':
2746                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2747                     return false
2748                 }
2749                 if (architecture != 'x64') {
2750                     return false
2751                 }
2752                 if (configuration != 'Checked') {
2753                     return false
2754                 }
2755                 break
2756             case 'longgc':
2757             case 'gcsimulator':
2758                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2759                     return false
2760                 }
2761                 if (architecture != 'x64') {
2762                     return false
2763                 }
2764                 if (configuration != 'Release') {
2765                     return false
2766                 }
2767                 break
2768             case 'gc_reliability_framework':
2769             case 'standalone_gc':
2770                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2771                     return false
2772                 }
2773
2774                 if (architecture != 'x64') {
2775                     return false
2776                 }
2777
2778                 if (configuration != 'Release' && configuration != 'Checked') {
2779                     return false
2780                 }
2781                 break
2782             // We only run Windows and Ubuntu x64 Checked for formatting right now
2783             case 'formatting':
2784                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2785                     return false
2786                 }
2787                 if (architecture != 'x64') {
2788                     return false
2789                 }
2790                 if (configuration != 'Checked') {
2791                     return false
2792                 }
2793                 break
2794             case 'illink':
2795                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2796                     return false
2797                 }
2798                 if (architecture != 'x64' && architecture != 'x86') {
2799                     return false
2800                 }
2801                 break
2802             case 'normal':
2803                 // Nothing skipped
2804                 break
2805             case 'innerloop':
2806                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2807                     return false
2808                 }
2809                 break
2810             case 'corefx_innerloop':
2811                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2812                     return false
2813                 }
2814                 if (architecture != 'x64') {
2815                     return false
2816                 }
2817                 break
2818             default:
2819                 println("Unknown scenario: ${scenario}")
2820                 assert false
2821                 break
2822         }
2823     }
2824
2825     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2826     // runtime behavior, not compile-time behavior, are not interesting.
2827     switch (architecture) {
2828         case 'x86_arm_altjit':
2829         case 'x64_arm64_altjit':
2830             if (isGCStressRelatedTesting(scenario)) {
2831                 return false
2832             }
2833             break
2834         default:
2835             break
2836     }
2837
2838     // The job was not filtered out, so we should generate it!
2839     return true
2840 }
2841
2842 Constants.allScenarios.each { scenario ->
2843     [true, false].each { isPR ->
2844         Constants.architectureList.each { architecture ->
2845             Constants.configurationList.each { configuration ->
2846                 Constants.osList.each { os ->
2847                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2848                     // and reset the os to Windows_NT
2849                     def isBuildOnly = false
2850                     if (os == 'Windows_NT_BuildOnly') {
2851                         isBuildOnly = true
2852                         os = 'Windows_NT'
2853                     }
2854
2855                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2856                         return
2857                     }
2858
2859                     // Calculate names
2860                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2861                     def folderName = getJobFolder(scenario)
2862
2863                     // Create the new job
2864                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2865
2866                     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
2867                     // "top-level" jobs. Build only jobs are such jobs.
2868                     if (!isBuildOnly)
2869                     {
2870                         addToViews(newJob, isPR, architecture, os)
2871                     }
2872
2873                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2874
2875                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2876                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2877                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2878
2879                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2880                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2881                     // artifacts produced.
2882                     if ((architecture == 'armem') && (os == 'Tizen')) {
2883                         // Define the Windows Tests and Corefx build job names
2884                         def lowerConfiguration = configuration.toLowerCase()
2885                         def WindowsTestsName = projectFolder + '/' +
2886                                                Utilities.getFullJobName(project,
2887                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2888                                                                         false)
2889                         def fxBranch = getFxBranch(branch)
2890                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2891                                            Utilities.getFolderName(fxBranch)
2892
2893                         def arm_abi = 'armel'
2894                         def corefx_os = 'tizen'
2895
2896                         // Let's use release CoreFX to test checked CoreCLR,
2897                         // because we do not generate checked CoreFX in CoreFX CI yet.
2898                         def corefx_lowerConfiguration = lowerConfiguration
2899                         if (lowerConfiguration == 'checked') {
2900                             corefx_lowerConfiguration = 'release'
2901                         }
2902
2903                         // Copy the Windows test binaries and the Corefx build binaries
2904                         newJob.with {
2905                             steps {
2906                                 copyArtifacts(WindowsTestsName) {
2907                                     includePatterns('bin/tests/tests.zip')
2908                                     buildSelector {
2909                                         latestSuccessful(true)
2910                                     }
2911                                 }
2912                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2913                                     includePatterns('bin/build.tar.gz')
2914                                     buildSelector {
2915                                         latestSuccessful(true)
2916                                     }
2917                                 }
2918                             } // steps
2919                         } // newJob.with
2920                     }
2921
2922                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2923
2924                     newJob.with {
2925                         steps {
2926                             if (os == 'Windows_NT') {
2927                                 buildCommands.each { buildCommand ->
2928                                     batchFile(buildCommand)
2929                                 }
2930                             }
2931                             else {
2932                                 buildCommands.each { buildCommand ->
2933                                     shell(buildCommand)
2934                                 }
2935                             }
2936                         } // steps
2937                     } // newJob.with
2938
2939                 } // os
2940             } // configuration
2941         } // architecture
2942     } // isPR
2943 } // scenario
2944
2945 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2946 // Returns the newly created job.
2947 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2948 {
2949     def osGroup = getOSGroup(os)
2950     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2951
2952     def jobFolder = getJobFolder(scenario)
2953     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2954         parameters {
2955             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2956         }
2957
2958         steps {
2959             // Set up the copies
2960
2961             // Coreclr build we are trying to test
2962             //
2963             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2964
2965             copyArtifacts(inputCoreCLRBuildName) {
2966                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2967                 buildSelector {
2968                     buildNumber('${CORECLR_BUILD}')
2969                 }
2970             }
2971
2972             if (isCoreFxScenario(scenario)) {
2973
2974                 // Only arm/arm64 supported for corefx testing now.
2975                 assert architecture == 'arm' || architecture == 'arm64'
2976
2977                 // Unzip CoreFx runtime
2978                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
2979
2980                 // Unzip CoreFx tests.
2981                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
2982
2983                 // Add the script to run the corefx tests
2984                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
2985                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2986                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
2987                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
2988
2989             } else { // !isCoreFxScenario(scenario)
2990
2991                 // Unzip tests.
2992                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
2993
2994                 def buildCommands = ""
2995
2996                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2997                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2998                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2999
3000                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3001                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3002
3003                 // For all jobs
3004                 addEnvVariable("CORE_ROOT", coreRootLocation)
3005                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3006                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3007
3008                 // If we are running a stress mode, we'll set those variables as well
3009                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3010                     def stressValues = null
3011                     if (isJitStressScenario(scenario)) {
3012                         stressValues = Constants.jitStressModeScenarios[scenario]
3013                     }
3014                     else {
3015                         stressValues = Constants.r2rStressScenarios[scenario]
3016                     }
3017
3018                     stressValues.each { key, value ->
3019                         addEnvVariable(key, value)
3020                     }
3021                 }
3022
3023                 if (isR2RScenario(scenario)) {
3024                     // Crossgen the framework assemblies.
3025                     buildCommands += """
3026 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3027 @goto skip_PrecompileAssembly
3028
3029 :PrecompileAssembly
3030 @REM Skip mscorlib since it is already precompiled.
3031 @if /I "%3" == "mscorlib.dll" exit /b 0
3032 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3033
3034 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3035 @if "%errorlevel%" == "-2146230517" (
3036     echo %2 is not a managed assembly.
3037 ) else if "%errorlevel%" == "-2146234344" (
3038     echo %2 is not a managed assembly.
3039 ) else if %errorlevel% neq 0 (
3040     echo Unable to precompile %2
3041 ) else (
3042     echo Precompiled %2
3043 )
3044 @exit /b 0
3045
3046 :skip_PrecompileAssembly
3047 """
3048
3049                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3050                     // crossgen on tests before running them.
3051                     addEnvVariable("RunCrossGen", "true")
3052                 } // isR2RScenario(scenario)
3053
3054                 // Create the smarty command
3055                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3056                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3057                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3058                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3059
3060                 // Exclude tests based on scenario.
3061                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3062                     addArchSpecificExclude(architecture, excludeTag)
3063                 }
3064
3065                 if (isInnerloopTestScenario(scenario)) {
3066                     addExclude("pri1")
3067                 }
3068
3069                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3070                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3071                 addExclude("LONG_RUNNING")
3072
3073                 smartyCommand += "/lstFile Tests.lst"
3074
3075                 def testListArch = [
3076                     'arm64': 'arm64',
3077                     'arm': 'arm'
3078                 ]
3079
3080                 def archLocation = testListArch[architecture]
3081
3082                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3083                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3084                 addCommand("${smartyCommand}")
3085
3086                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3087                 // However, we also need to remove all the variables that were set during this batch file, so we
3088                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3089                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3090                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3091                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3092                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3093                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3094
3095                 // ZIP up the smarty output, no matter what the smarty result.
3096                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3097
3098                 addCommand("echo %errorlevel%")
3099                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3100
3101                 // Use the smarty errorlevel as the script errorlevel.
3102                 addCommand("exit /b %__save_smarty_errorlevel%")
3103
3104                 batchFile(buildCommands)
3105             } // non-corefx testing
3106         } // steps
3107     } // job
3108
3109     if (!isCoreFxScenario(scenario)) {
3110         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3111
3112         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3113         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3114         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3115         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3116     }
3117
3118     return newJob
3119 }
3120
3121 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3122 // E.g., non-Windows tests.
3123 // Returns the newly created job.
3124 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3125 {
3126     def lowerConfiguration = configuration.toLowerCase()
3127
3128     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3129     def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3130     def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3131
3132     def doCoreFxTesting = isCoreFxScenario(scenario)
3133
3134     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3135
3136     def osGroup = getOSGroup(os)
3137     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3138
3139     def testOpts = ''
3140     def useServerGC = false
3141
3142     // Enable Server GC for Ubuntu PR builds
3143     // REVIEW: why? Does this apply to all architectures? Why only PR?
3144     if (os == 'Ubuntu' && isPR) {
3145         testOpts += ' --useServerGC'
3146         useServerGC = true
3147     }
3148
3149     if (isR2RScenario(scenario)) {
3150
3151         testOpts += ' --crossgen --runcrossgentests'
3152
3153         if (scenario == 'r2r_jitstress1') {
3154             testOpts += ' --jitstress=1'
3155         }
3156         else if (scenario == 'r2r_jitstress2') {
3157             testOpts += ' --jitstress=2'
3158         }
3159         else if (scenario == 'r2r_jitstress1_tiered') {
3160             testOpts += ' --jitstress=1'
3161         }
3162         else if (scenario == 'r2r_jitstress2_tiered') {
3163             testOpts += ' --jitstress=2'
3164         }
3165         else if (scenario == 'r2r_jitstressregs1') {
3166             testOpts += ' --jitstressregs=1'
3167         }
3168         else if (scenario == 'r2r_jitstressregs2') {
3169             testOpts += ' --jitstressregs=2'
3170         }
3171         else if (scenario == 'r2r_jitstressregs3') {
3172             testOpts += ' --jitstressregs=3'
3173         }
3174         else if (scenario == 'r2r_jitstressregs4') {
3175             testOpts += ' --jitstressregs=4'
3176         }
3177         else if (scenario == 'r2r_jitstressregs8') {
3178             testOpts += ' --jitstressregs=8'
3179         }
3180         else if (scenario == 'r2r_jitstressregs0x10') {
3181             testOpts += ' --jitstressregs=0x10'
3182         }
3183         else if (scenario == 'r2r_jitstressregs0x80') {
3184             testOpts += ' --jitstressregs=0x80'
3185         }
3186         else if (scenario == 'r2r_jitstressregs0x1000') {
3187             testOpts += ' --jitstressregs=0x1000'
3188         }
3189         else if (scenario == 'r2r_jitminopts') {
3190             testOpts += ' --jitminopts'
3191         }
3192         else if (scenario == 'r2r_jitforcerelocs') {
3193             testOpts += ' --jitforcerelocs'
3194         }
3195         else if (scenario == 'r2r_gcstress15') {
3196             testOpts += ' --gcstresslevel=0xF'
3197         }
3198     }
3199     else if (scenario == 'jitdiff') {
3200         testOpts += ' --jitdisasm --crossgen'
3201     }
3202     else if (scenario == 'illink') {
3203         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3204     }
3205     else if (isLongGc(scenario)) {
3206         // Long GC tests behave very poorly when they are not
3207         // the only test running (many of them allocate until OOM).
3208         testOpts += ' --sequential'
3209
3210         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3211         // for running long GC and GCSimulator tests, respectively. We don't use them
3212         // here because using a playlist file produces much more readable output on the CI machines
3213         // and reduces running time.
3214         //
3215         // The Long GC playlist contains all of the tests that are
3216         // going to be run. The GCSimulator playlist contains all of
3217         // the GC simulator tests.
3218         if (scenario == 'longgc') {
3219             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3220         }
3221         else if (scenario == 'gcsimulator') {
3222             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3223         }
3224     }
3225     else if (isGcReliabilityFramework(scenario)) {
3226         testOpts += ' --build-overlay-only'
3227     }
3228     else if (scenario == 'standalone_gc') {
3229         if (osGroup == 'OSX') {
3230             testOpts += ' --gcname=libclrgc.dylib'
3231         }
3232         else if (osGroup == 'Linux') {
3233             testOpts += ' --gcname=libclrgc.so'
3234         }
3235         else {
3236             println("Unexpected OS group: ${osGroup} for os ${os}")
3237             assert false
3238         }
3239     }
3240
3241     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3242     // will be null in this case.
3243
3244     def jobFolder = getJobFolder(scenario)
3245     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3246         parameters {
3247             if (inputTestsBuildName != null) {
3248                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3249             }
3250             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3251         }
3252
3253         steps {
3254             // Set up the copies
3255
3256             // Coreclr build containing the tests and mscorlib
3257             // pri1 jobs still need to copy windows_nt built tests
3258             if (inputTestsBuildName != null) {
3259                 copyArtifacts(inputTestsBuildName) {
3260                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3261                     buildSelector {
3262                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3263                     }
3264                 }
3265             }
3266
3267             // Coreclr build we are trying to test
3268             //
3269             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3270             //
3271             // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3272             // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3273
3274             if (!isUbuntuArm64Job) {
3275                 copyArtifacts(inputCoreCLRBuildName) {
3276                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3277                     buildSelector {
3278                         buildNumber('${CORECLR_BUILD}')
3279                     }
3280                 }
3281             }
3282
3283             if (isUbuntuArmJob) {
3284                 // Add some useful information to the log file. Ignore return codes.
3285                 shell("uname -a || true")
3286             }
3287
3288             if (isUbuntuArm64Job) {
3289                 // Copy the required artifacts directly, using wget, e.g.:
3290                 //
3291                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3292                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3293                 //
3294                 // parameterized as:
3295                 //
3296                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3297                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3298                 //
3299                 // CoreFX example artifact URLs:
3300                 //
3301                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3302                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3303                 //
3304                 // Note that the source might be in a "jitstress" folder.
3305                 //
3306                 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3307                 //
3308                 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3309
3310                 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3311
3312                 def mungedProjectName = Utilities.getFolderName(project)
3313                 def mungedBranchName = Utilities.getFolderName(branch)
3314
3315                 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3316                 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3317                 if (doCoreFxTesting || doCrossGenComparison) {
3318                     // These depend on unique builds for each scenario
3319                     inputCoreCLRBuildScenario = scenario
3320                 }
3321                 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3322                 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3323
3324                 // Need to add the sub-folder if necessary.
3325                 def inputJobPath = "job/${inputJobName}"
3326                 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3327                 if (folderName != '') {
3328                     inputJobPath = "job/${folderName}/job/${inputJobName}"
3329                 }
3330
3331                 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3332
3333                 if (doCoreFxTesting) {
3334                     shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3335                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3336                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3337                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/run-test.sh")
3338                     shell("chmod +x ${workspaceRelativeFxRootLinux}/run-test.sh")
3339                 }
3340                 else {
3341                     shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
3342                     shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
3343                 }
3344             }
3345
3346             if (architecture == 'x86') {
3347                 shell("mkdir ./bin/CoreFxNative")
3348
3349                 def fxBranch = getFxBranch(branch)
3350                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3351
3352                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3353                     includePatterns('bin/build.tar.gz')
3354                     targetDirectory('bin/CoreFxNative')
3355                     buildSelector {
3356                         latestSuccessful(true)
3357                     }
3358                 }
3359
3360                 shell("mkdir ./bin/CoreFxBinDir")
3361                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3362             }
3363
3364             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3365             if (doCoreFxTesting) {
3366                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3367                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3368             }
3369             else if (architecture != 'arm64') {
3370                 // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
3371                 //
3372                 // Unzip the tests first.  Exit with 0
3373                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3374                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3375             }
3376
3377             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3378             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3379             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3380             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3381             // copied correctly.
3382             if (!doCoreFxTesting) {
3383                 if (isUbuntuArmJob) {
3384                     if (architecture == 'arm') {
3385                         shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3386                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3387                     }
3388                     else {
3389                         assert architecture == 'arm64'
3390                         shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
3391
3392                         // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
3393                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3394                     }
3395                 }
3396                 else {
3397                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3398                 }
3399             }
3400
3401             // Execute the tests
3402             def runDocker = isNeedDocker(architecture, os, false)
3403             def dockerPrefix = ""
3404             def dockerCmd = ""
3405             if (runDocker) {
3406                 def dockerImage = getDockerImageName(architecture, os, false)
3407                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3408                 dockerCmd = dockerPrefix + "${dockerImage} "
3409             }
3410
3411             // If we are running a stress mode, we'll set those variables first.
3412             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3413             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3414                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3415                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3416                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3417                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3418                 shell("${envScriptCmds}")
3419                 testOpts += " --test-env=${scriptFileName}"
3420             }
3421
3422             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3423             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3424             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3425             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3426             if (isGCStressRelatedTesting(scenario)) {
3427                 if (architecture == 'x64') {
3428                     shell('./init-tools.sh')
3429                 }
3430             }
3431
3432             if (doCoreFxTesting) {
3433                 shell("""\
3434 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3435             }
3436             else {
3437                 def runScript = "${dockerCmd}./tests/runtest.sh"
3438
3439                 // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
3440
3441                 shell("""\
3442 ${runScript} \\
3443     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3444     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3445     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3446     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3447             }
3448
3449             if (isGcReliabilityFramework(scenario)) {
3450                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3451                 if (useServerGC) {
3452                     if (runDocker) {
3453                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3454                     }
3455                     else {
3456                         shell("export COMPlus_gcServer=1")
3457                     }
3458                 }
3459
3460                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3461             }
3462         } // steps
3463     } // job
3464
3465     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3466     if (os in ['Ubuntu']) {
3467         SummaryBuilder summaries = new SummaryBuilder()
3468         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3469         summaries.emit(newJob)
3470     }
3471
3472     if (doCoreFxTesting) {
3473         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3474         if ((os == "Ubuntu") && (architecture == 'arm')) {
3475             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3476             //
3477             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3478             //
3479             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3480             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3481         }
3482         else {
3483             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3484         }
3485     }
3486     else {
3487         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3488     }
3489
3490     return newJob
3491 }
3492
3493 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3494 {
3495     assert isCrossGenComparisonScenario(scenario)
3496
3497     def osGroup = getOSGroup(os)
3498     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3499
3500     def workspaceRelativeResultsDir = "_"
3501     def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3502
3503     def jobFolder = getJobFolder(scenario)
3504     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3505         parameters {
3506             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3507         }
3508
3509         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3510
3511         steps {
3512             copyArtifacts(inputCoreCLRBuildName) {
3513                 includePatterns("${workspaceRelativeArtifactsArchive}")
3514                 buildSelector {
3515                     buildNumber('${CORECLR_BUILD}')
3516                 }
3517             }
3518
3519             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3520
3521             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3522             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3523             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3524             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3525
3526             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3527             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3528
3529             shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3530             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3531             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3532
3533             getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3534                 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3535                 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3536             } // crossArch
3537         } // steps
3538     }  // job
3539
3540     Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**")
3541     getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3542         def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3543         Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**")
3544     } // crossArch
3545
3546     return newJob
3547 }
3548
3549 // Create a test job that will be used by a flow job.
3550 // Returns the newly created job.
3551 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3552 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3553 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3554 {
3555     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3556
3557     def newJob = null
3558     if (windowsArmJob) {
3559         assert inputTestsBuildName == null
3560         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3561     }
3562     else if (isCrossGenComparisonScenario(scenario)) {
3563         assert inputTestsBuildName == null
3564         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3565     }
3566     else {
3567         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3568     }
3569
3570     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3571
3572     if (scenario == 'jitdiff') {
3573         def osGroup = getOSGroup(os)
3574         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3575     }
3576
3577     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3578     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3579
3580     return newJob
3581 }
3582
3583 // Create a flow job to tie together a build job with the given test job.
3584 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3585 // Returns the new flow job.
3586 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3587 {
3588     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3589     // Linux CoreCLR test
3590     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3591     def jobFolder = getJobFolder(scenario)
3592
3593     def newFlowJob = null
3594
3595     if (inputTestsBuildName == null) {
3596         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3597                         buildFlow("""\
3598 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3599
3600 // And then build the test build
3601 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3602 """)
3603         }
3604         JobReport.Report.addReference(inputCoreCLRBuildName)
3605         JobReport.Report.addReference(fullTestJobName)
3606     }
3607     else {
3608         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3609                         buildFlow("""\
3610 // Build the input jobs in parallel
3611 parallel (
3612 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3613 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3614 )
3615
3616 // And then build the test build
3617 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3618                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3619 """)
3620         }
3621         JobReport.Report.addReference(inputCoreCLRBuildName)
3622         JobReport.Report.addReference(inputTestsBuildName)
3623         JobReport.Report.addReference(fullTestJobName)
3624     }
3625
3626     addToViews(newFlowJob, isPR, architecture, os)
3627
3628     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3629
3630     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3631     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3632
3633     return newFlowJob
3634 }
3635
3636 // Determine if we should generate a flow job for the given parameters.
3637 // Returns true if the job should be generated.
3638 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3639 {
3640     // The various "innerloop" jobs are only available as PR triggered.
3641
3642     if (!isPR) {
3643         if (isInnerloopTestScenario(scenario)) {
3644             return false
3645         }
3646
3647         if (scenario == 'corefx_innerloop') {
3648             return false
3649         }
3650     }
3651
3652     // Filter based on OS and architecture.
3653
3654     switch (architecture) {
3655         case 'arm':
3656             if (os != "Ubuntu" && os != "Windows_NT") {
3657                 return false
3658             }
3659             break
3660         case 'arm64':
3661             if (os != "Ubuntu16.04" && os != "Windows_NT") {
3662                 return false
3663             }
3664             break
3665         case 'x86':
3666             if (os != "Ubuntu") {
3667                 return false
3668             }
3669             break
3670         case 'x64':
3671             if (!(os in Constants.crossList)) {
3672                 return false
3673             }
3674             if (os == "Windows_NT") {
3675                 return false
3676             }
3677             break
3678         case 'armem':
3679         case 'x86_arm_altjit':
3680         case 'x64_arm64_altjit':
3681             // No flow jobs
3682             return false
3683         default:
3684             println("Unknown architecture: ${architecture}")
3685             assert false
3686             break
3687     }
3688
3689     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3690
3691     // Filter based on scenario in OS.
3692
3693     if (os == 'Windows_NT') {
3694         assert architecture == 'arm' || architecture == 'arm64'
3695         if (!isArmWindowsScenario(scenario)) {
3696             return false
3697         }
3698         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3699             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3700             // To do that, it doesn't require a flow job.
3701             return false
3702         }
3703     }
3704     else {
3705         // Non-Windows
3706         if (architecture == 'arm') {
3707             if (!(scenario in Constants.validLinuxArmScenarios)) {
3708                 return false
3709             }
3710         }
3711         else if (architecture == 'arm64') {
3712             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3713                 return false
3714             }
3715         }
3716         else if (architecture == 'x86') {
3717             // Linux/x86 only want innerloop and default test
3718             if (!isNormalOrInnerloop) {
3719                 return false
3720             }
3721         }
3722         else if (architecture == 'x64') {
3723             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3724             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3725             // build from a Windows machine.
3726             if (isCoreFxScenario(scenario)) {
3727                 return false
3728             }
3729         }
3730     }
3731
3732     // For CentOS, we only want Checked/Release builds.
3733     if (os == 'CentOS7.1') {
3734         if (configuration != 'Checked' && configuration != 'Release') {
3735             return false
3736         }
3737         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3738             return false
3739         }
3740     }
3741
3742     // For RedHat and Debian, we only do Release builds.
3743     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3744         if (configuration != 'Release') {
3745             return false
3746         }
3747         if (!isNormalOrInnerloop) {
3748             return false
3749         }
3750     }
3751
3752     // Next, filter based on scenario.
3753
3754     if (isJitStressScenario(scenario)) {
3755         if (configuration != 'Checked') {
3756             return false
3757         }
3758     }
3759     else if (isR2RBaselineScenario(scenario)) {
3760         if (configuration != 'Checked' && configuration != 'Release') {
3761             return false
3762         }
3763     }
3764     else if (isR2RStressScenario(scenario)) {
3765         if (configuration != 'Checked') {
3766             return false
3767         }
3768     }
3769     else if (isCrossGenComparisonScenario(scenario)) {
3770         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3771     }
3772     else {
3773         // Skip scenarios
3774         switch (scenario) {
3775             case 'ilrt':
3776             case 'longgc':
3777             case 'gcsimulator':
3778                 // Long GC tests take a long time on non-Release builds
3779                 // ilrt is also Release only
3780                 if (configuration != 'Release') {
3781                     return false
3782                 }
3783                 break
3784
3785             case 'jitdiff':
3786                 if (configuration != 'Checked') {
3787                     return false
3788                 }
3789                 break
3790
3791             case 'gc_reliability_framework':
3792             case 'standalone_gc':
3793                 if (configuration != 'Release' && configuration != 'Checked') {
3794                     return false
3795                 }
3796                 break
3797
3798             case 'formatting':
3799                 return false
3800
3801             case 'illink':
3802                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3803                     return false
3804                 }
3805                 break
3806
3807             case 'normal':
3808                 // Nothing skipped
3809                 break
3810
3811             case 'innerloop':
3812                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3813                     return false
3814                 }
3815                 break
3816
3817             case 'corefx_innerloop':
3818                 // No flow job needed
3819                 return false
3820
3821             default:
3822                 println("Unknown scenario: ${scenario}")
3823                 assert false
3824                 break
3825         }
3826     }
3827
3828     // The job was not filtered out, so we should generate it!
3829     return true
3830 }
3831
3832 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3833 Constants.allScenarios.each { scenario ->
3834     [true, false].each { isPR ->
3835         Constants.architectureList.each { architecture ->
3836             Constants.configurationList.each { configuration ->
3837                 Constants.osList.each { os ->
3838
3839                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3840                         return
3841                     }
3842
3843                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3844                     def doCoreFxTesting = isCoreFxScenario(scenario)
3845                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3846
3847                     // Figure out the job name of the CoreCLR build the test will depend on.
3848
3849                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3850                     def inputCoreCLRBuildIsBuildOnly = false
3851                     if (doCoreFxTesting) {
3852                         // Every CoreFx test depends on its own unique build.
3853                         inputCoreCLRBuildScenario = scenario
3854                         if (windowsArmJob) {
3855                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3856                             inputCoreCLRBuildIsBuildOnly = true
3857                         }
3858                     }
3859                     if (doCrossGenComparison) {
3860                         inputCoreCLRBuildScenario = scenario
3861                     }
3862
3863                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3864                     def inputCoreCLRBuildName = projectFolder + '/' +
3865                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3866
3867                     // Figure out the name of the build job that the test job will depend on.
3868                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3869                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3870                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3871                     // CoreFX build creates the tests.
3872
3873                     def inputTestsBuildName = null
3874
3875                     // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
3876                     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3877
3878                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job) {
3879                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3880
3881                         def inputTestsBuildArch = architecture
3882                         if (architecture == "arm") {
3883                             // Use the x86 test build for arm unix
3884                             inputTestsBuildArch = "x86"
3885                         }
3886
3887                         def inputTestsBuildIsBuildOnly = true
3888
3889                         inputTestsBuildName = projectFolder + '/' +
3890                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3891                     }
3892
3893                     // =============================================================================================
3894                     // Create the test job
3895                     // =============================================================================================
3896
3897                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3898
3899                     // =============================================================================================
3900                     // Create a build flow to join together the build and tests required to run this test.
3901                     // =============================================================================================
3902
3903                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3904                         // Do not create the flow job for RHEL jobs.
3905                         return
3906                     }
3907
3908                     def fullTestJobName = projectFolder + '/' + testJob.name
3909                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3910
3911                 } // os
3912             } // configuration
3913         } // architecture
3914     } // isPR
3915 } // scenario
3916
3917 JobReport.Report.generateJobReport(out)
3918
3919 // Make the call to generate the help job
3920 Utilities.createHelperJob(this, project, branch,
3921     "Welcome to the ${project} Repository",  // This is prepended to the help message
3922     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3923
3924 Utilities.addCROSSCheck(this, project, branch)