netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu'      : 'Linux',
  21                       'Ubuntu16.04' : 'Linux',
  22                       'Ubuntu16.10' : 'Linux',
  23                       'RHEL7.2'     : 'Linux',
  24                       'Debian8.4'   : 'Linux',
  25                       'Fedora24'    : 'Linux',
  26                       'CentOS7.1'   : 'Linux',
  27                       'Tizen'       : 'Linux',
  28                       'OSX10.12'    : 'OSX',
  29                       'Windows_NT'  : 'Windows_NT']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitectures(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return ['x86','x64']
  39
  40         case 'arm64':
  41             return ['x64']
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited Windows ARM64 hardware (used for ARM/ARM64 testing) and Linux/arm32 and Linux/arm64 hardware.
  51     // So only allow certain branches to use it.
  52     def static LimitedHardwareBranches = [
  53                'master']
  54
  55     // Innerloop build OS's
  56     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  57     // test execution in the build flow runs.  It generates the exact same build
  58     // as Windows_NT but without running the tests.
  59     def static osList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'Windows_NT_BuildOnly',
  65                'CentOS7.1',
  66                'RHEL7.2',
  67                'Ubuntu16.04',
  68                'Ubuntu16.10',
  69                'Fedora24',
  70                'Tizen']
  71
  72     def static crossList = [
  73                'Ubuntu',
  74                'Debian8.4',
  75                'OSX10.12',
  76                'Windows_NT',
  77                'CentOS7.1',
  78                'RHEL7.2']
  79
  80     // This is a set of JIT stress modes combined with the set of variables that
  81     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  82     // the values are the environment variables
  83     def static jitStressModeScenarios = [
  84                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  85                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  86                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  87                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  88                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  89                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  90                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  91                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  92                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  93                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  94                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  95                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  96                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  97                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  98                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  99                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 100                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 101                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 102                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 103                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 104                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 105                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 106                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 107                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 108                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 109                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 110                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 111                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 112                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 113                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 114                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 115                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 116                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 117                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 118                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 119                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 120                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 121                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 122                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 123                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 124                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 125                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 126                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 127                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 128                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 129                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 130                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 131                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 132                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 133                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 134                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 135                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 136                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 137                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 138                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 139                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 140     ]
 141
 142     // This is a set of ReadyToRun stress scenarios
 143     def static r2rStressScenarios = [
 144                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 145                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 146                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 147                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 148                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 149                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 150                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 151                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 152                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 153                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 154                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 155                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 156                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 157                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 158                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 159                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 160     ]
 161
 162     // This is the basic set of scenarios
 163     def static basicScenarios = [
 164                'innerloop',
 165                'normal',
 166                'ilrt',
 167                'r2r',
 168                'longgc',
 169                'formatting',
 170                'gcsimulator',
 171                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 172                'standalone_gc',
 173                'gc_reliability_framework',
 174                'illink',
 175                'corefx_innerloop',
 176                'crossgen_comparison']
 177
 178     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 179
 180     // Valid PR trigger combinations.
 181     def static prTriggeredValidInnerLoopCombos = [
 182         'Windows_NT': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'x86': [
 187                 'Checked',
 188                 'Release'
 189             ],
 190             'arm': [
 191                 'Debug',
 192                 'Checked'
 193             ],
 194             'arm64': [
 195                 'Debug',
 196                 'Checked'
 197             ]
 198         ],
 199         'Windows_NT_BuildOnly': [
 200             'x64': [
 201                 'Checked',
 202                 'Release'
 203             ],
 204             'x86': [
 205                 'Checked',
 206                 'Release'
 207             ],
 208             'arm': [
 209                 'Checked'
 210             ],
 211         ],
 212         'Ubuntu': [
 213             'x64': [
 214                 'Checked'
 215             ],
 216             'arm': [
 217                 'Checked'
 218             ]
 219         ],
 220         'Ubuntu16.04': [
 221             'arm64': [
 222                 'Checked'
 223             ]
 224         ],
 225         'CentOS7.1': [
 226             'x64': [
 227                 'Debug',
 228                 'Checked'
 229             ]
 230         ],
 231         'OSX10.12': [
 232             'x64': [
 233                 'Checked'
 234             ]
 235         ],
 236         'Tizen': [
 237             'armem': [
 238                 'Checked'
 239             ]
 240         ]
 241     ]
 242
 243     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 244     // to Tests.lst file categories to exclude.
 245     //
 246     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 247     // and with the same values, with some commented out, for easier maintenance.
 248     //
 249     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 250     //
 251     def static validArmWindowsScenarios = [
 252                'innerloop':                              [],
 253                'normal':                                 [],
 254                // 'ilrt'
 255                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 256                // 'longgc'
 257                // 'formatting'
 258                // 'gcsimulator'
 259                // 'jitdiff'
 260                // 'standalone_gc'
 261                // 'gc_reliability_framework'
 262                // 'illink'
 263                // 'corefx_innerloop'
 264                // 'crossgen_comparison'
 265                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 277                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 278                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 279                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 280                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 281                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 282                'tieredcompilation':                      [],
 283                'no_tiered_compilation':                  [],
 284                'no_tiered_compilation_innerloop':        [],
 285                'forcerelocs':                            [],
 286                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 290                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 293                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 294                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 307                // 'jitsse2only'                          // Only relevant to xarch
 308                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 309                // 'jitincompletehwintrinsic'
 310                // 'jitx86hwintrinsicnoavx'
 311                // 'jitx86hwintrinsicnoavx2'
 312                // 'jitx86hwintrinsicnosimd'
 313                // 'jitnox86hwintrinsic'
 314                'corefx_baseline':                        [],    // corefx tests don't use smarty
 315                'corefx_minopts':                         [],    // corefx tests don't use smarty
 316                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 317                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 318                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 319                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 320                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 321                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 322                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 323                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 324                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 325                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 326                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 327                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 328                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 329                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 330                'heapverify1':                            [],
 331                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 332                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 333                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 334                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 336                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 337
 338                //
 339                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 340                //
 341
 342                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 343                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 344                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 345                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 346                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 347                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 348                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 349                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 350                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 351                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 352                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 353                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 354                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 355                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 356                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 357                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 358                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 359                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 360                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 361     ]
 362
 363     def static validLinuxArmScenarios = [
 364                'innerloop',
 365                'normal',
 366                // 'ilrt'
 367                'r2r',
 368                // 'longgc'
 369                // 'formatting'
 370                // 'gcsimulator'
 371                // 'jitdiff'
 372                // 'standalone_gc'
 373                // 'gc_reliability_framework'
 374                // 'illink'
 375                // 'corefx_innerloop'
 376                'crossgen_comparison',
 377                'r2r_jitstress1',
 378                'r2r_jitstress2',
 379                'r2r_jitstress1_tiered',
 380                'r2r_jitstress2_tiered',
 381                'r2r_jitstressregs1',
 382                'r2r_jitstressregs2',
 383                'r2r_jitstressregs3',
 384                'r2r_jitstressregs4',
 385                'r2r_jitstressregs8',
 386                'r2r_jitstressregs0x10',
 387                'r2r_jitstressregs0x80',
 388                'r2r_jitstressregs0x1000',
 389                'r2r_jitminopts',
 390                'r2r_jitforcerelocs',
 391                'r2r_gcstress15',
 392                'r2r_no_tiered_compilation',
 393                'minopts',
 394                'tieredcompilation',
 395                'no_tiered_compilation',
 396                'no_tiered_compilation_innerloop',
 397                'forcerelocs',
 398                'jitstress1',
 399                'jitstress2',
 400                'jitstress1_tiered',
 401                'jitstress2_tiered',
 402                'jitstressregs1',
 403                'jitstressregs2',
 404                'jitstressregs3',
 405                'jitstressregs4',
 406                'jitstressregs8',
 407                'jitstressregs0x10',
 408                'jitstressregs0x80',
 409                'jitstressregs0x1000',
 410                'jitstress2_jitstressregs1',
 411                'jitstress2_jitstressregs2',
 412                'jitstress2_jitstressregs3',
 413                'jitstress2_jitstressregs4',
 414                'jitstress2_jitstressregs8',
 415                'jitstress2_jitstressregs0x10',
 416                'jitstress2_jitstressregs0x80',
 417                'jitstress2_jitstressregs0x1000',
 418                'tailcallstress',
 419                // 'jitsse2only'                          // Only relevant to xarch
 420                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 421                // 'jitincompletehwintrinsic'
 422                // 'jitx86hwintrinsicnoavx'
 423                // 'jitx86hwintrinsicnoavx2'
 424                // 'jitx86hwintrinsicnosimd'
 425                // 'jitnox86hwintrinsic'
 426                'corefx_baseline',
 427                'corefx_minopts',
 428                'corefx_tieredcompilation',
 429                'corefx_jitstress1',
 430                'corefx_jitstress2',
 431                'corefx_jitstressregs1',
 432                'corefx_jitstressregs2',
 433                'corefx_jitstressregs3',
 434                'corefx_jitstressregs4',
 435                'corefx_jitstressregs8',
 436                'corefx_jitstressregs0x10',
 437                'corefx_jitstressregs0x80',
 438                'corefx_jitstressregs0x1000',
 439                'gcstress0x3',
 440                'gcstress0xc',
 441                'zapdisable',
 442                'heapverify1',
 443                'gcstress0xc_zapdisable',
 444                'gcstress0xc_zapdisable_jitstress2',
 445                'gcstress0xc_zapdisable_heapverify1',
 446                'gcstress0xc_jitstress1',
 447                'gcstress0xc_jitstress2',
 448                'gcstress0xc_minopts_heapverify1'
 449     ]
 450
 451     def static validLinuxArm64Scenarios = [
 452                'innerloop',
 453                'normal',
 454                // 'ilrt'
 455                'r2r',
 456                // 'longgc'
 457                // 'formatting'
 458                // 'gcsimulator'
 459                // 'jitdiff'
 460                // 'standalone_gc'
 461                // 'gc_reliability_framework'
 462                // 'illink'
 463                // 'corefx_innerloop'
 464                // 'crossgen_comparison'
 465                'r2r_jitstress1',
 466                'r2r_jitstress2',
 467                'r2r_jitstress1_tiered',
 468                'r2r_jitstress2_tiered',
 469                'r2r_jitstressregs1',
 470                'r2r_jitstressregs2',
 471                'r2r_jitstressregs3',
 472                'r2r_jitstressregs4',
 473                'r2r_jitstressregs8',
 474                'r2r_jitstressregs0x10',
 475                'r2r_jitstressregs0x80',
 476                'r2r_jitstressregs0x1000',
 477                'r2r_jitminopts',
 478                'r2r_jitforcerelocs',
 479                'r2r_gcstress15',
 480                'r2r_no_tiered_compilation',
 481                'minopts',
 482                'tieredcompilation',
 483                'no_tiered_compilation',
 484                'no_tiered_compilation_innerloop',
 485                'forcerelocs',
 486                'jitstress1',
 487                'jitstress2',
 488                'jitstress1_tiered',
 489                'jitstress2_tiered',
 490                'jitstressregs1',
 491                'jitstressregs2',
 492                'jitstressregs3',
 493                'jitstressregs4',
 494                'jitstressregs8',
 495                'jitstressregs0x10',
 496                'jitstressregs0x80',
 497                'jitstressregs0x1000',
 498                'jitstress2_jitstressregs1',
 499                'jitstress2_jitstressregs2',
 500                'jitstress2_jitstressregs3',
 501                'jitstress2_jitstressregs4',
 502                'jitstress2_jitstressregs8',
 503                'jitstress2_jitstressregs0x10',
 504                'jitstress2_jitstressregs0x80',
 505                'jitstress2_jitstressregs0x1000',
 506                'tailcallstress',
 507                // 'jitsse2only'                         // Only relevant to xarch
 508                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 509                // 'jitincompletehwintrinsic'
 510                // 'jitx86hwintrinsicnoavx'
 511                // 'jitx86hwintrinsicnoavx2'
 512                // 'jitx86hwintrinsicnosimd'
 513                // 'jitnox86hwintrinsic'
 514                'corefx_baseline',
 515                'corefx_minopts',
 516                'corefx_tieredcompilation',
 517                'corefx_jitstress1',
 518                'corefx_jitstress2',
 519                'corefx_jitstressregs1',
 520                'corefx_jitstressregs2',
 521                'corefx_jitstressregs3',
 522                'corefx_jitstressregs4',
 523                'corefx_jitstressregs8',
 524                'corefx_jitstressregs0x10',
 525                'corefx_jitstressregs0x80',
 526                'corefx_jitstressregs0x1000',
 527                'gcstress0x3',
 528                'gcstress0xc',
 529                'zapdisable',
 530                'heapverify1',
 531                'gcstress0xc_zapdisable',
 532                'gcstress0xc_zapdisable_jitstress2',
 533                'gcstress0xc_zapdisable_heapverify1',
 534                'gcstress0xc_jitstress1',
 535                'gcstress0xc_jitstress2',
 536                'gcstress0xc_minopts_heapverify1'
 537     ]
 538
 539     def static configurationList = ['Debug', 'Checked', 'Release']
 540
 541     // This is the set of architectures
 542     // Some of these are pseudo-architectures:
 543     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 544     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 545     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 546     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 547
 548     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 549     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 550 }
 551
 552 // **************************************************************
 553 // Create some specific views
 554 //
 555 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 556 // views based on a single regular expression. These views will be generated by adding a
 557 // specific set of jobs to them.
 558 //
 559 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 560 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 561 // those are required, don't add them (which simplifies the view pages, as well).
 562 // **************************************************************
 563
 564 class Views {
 565     def static MergeJobView = null
 566     def static PeriodicJobView = null
 567     def static ArchitectureViews = [:]
 568     def static OSViews = [:]
 569 }
 570
 571 // MergeJobView: include all jobs that execute when a PR change is merged.
 572 Views.MergeJobView = listView('Merge') {
 573     recurse()
 574     columns {
 575         status()
 576         weather()
 577         name()
 578         lastSuccess()
 579         lastFailure()
 580         lastDuration()
 581         buildButton()
 582     }
 583 }
 584
 585 // PeriodicJobView: include all jobs that execute on a schedule
 586 Views.PeriodicJobView = listView('Periodic') {
 587     recurse()
 588     columns {
 589         status()
 590         weather()
 591         name()
 592         lastSuccess()
 593         lastFailure()
 594         lastDuration()
 595         buildButton()
 596     }
 597 }
 598
 599 // Create a view for non-PR jobs for each architecture.
 600 Constants.architectureList.each { architecture ->
 601     Views.ArchitectureViews[architecture] = listView(architecture) {
 602         recurse()
 603         columns {
 604             status()
 605             weather()
 606             name()
 607             lastSuccess()
 608             lastFailure()
 609             lastDuration()
 610             buildButton()
 611         }
 612     }
 613 }
 614
 615 // Create a view for non-PR jobs for each OS.
 616 Constants.osList.each { os ->
 617     // Don't create one for the special 'Windows_NT_BuildOnly'
 618     if (os == 'Windows_NT_BuildOnly') {
 619         return
 620     }
 621     Views.OSViews[os] = listView(os) {
 622         recurse()
 623         columns {
 624             status()
 625             weather()
 626             name()
 627             lastSuccess()
 628             lastFailure()
 629             lastDuration()
 630             buildButton()
 631         }
 632     }
 633 }
 634
 635 def static addToMergeView(def job) {
 636     Views.MergeJobView.with {
 637         jobs {
 638             name(job.name)
 639         }
 640     }
 641 }
 642
 643 def static addToPeriodicView(def job) {
 644     Views.PeriodicJobView.with {
 645         jobs {
 646             name(job.name)
 647         }
 648     }
 649 }
 650
 651 def static addToViews(def job, def isFlowJob, def isPR, def architecture, def os, def configuration, def scenario) {
 652     if (isPR) {
 653         // No views want PR jobs currently.
 654         return
 655     }
 656
 657     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
 658     // "top-level" jobs. Build only jobs are such jobs.
 659     if (os == 'Windows_NT_BuildOnly') {
 660         return
 661     }
 662
 663     if (!isFlowJob) {
 664         // For non-flow jobs, which ones are only used by flow jobs?
 665         if ((architecture == 'arm') || (architecture == 'arm64')) {
 666             if (isCoreFxScenario(scenario)) {
 667                 // We have corefx-specific scenario builds for each of the runs, but these are driven by flow jobs.
 668                 return
 669             }
 670
 671             // We're left with the basic normal/innerloop builds. We might want to remove these from the views also, if desired.
 672             // However, there are a few, like the Debug Build, that is build only, not "Build and Test", that we should leave.
 673         }
 674     }
 675
 676     // Add to architecture view.
 677     Views.ArchitectureViews[architecture].with {
 678         jobs {
 679             name(job.name)
 680         }
 681     }
 682
 683     // Add to OS view.
 684     Views.OSViews[os].with {
 685         jobs {
 686             name(job.name)
 687         }
 688     }
 689 }
 690
 691 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 692     addToPeriodicView(job)
 693     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 694 }
 695
 696 def static addGithubPushTriggerHelper(def job) {
 697     addToMergeView(job)
 698     Utilities.addGithubPushTrigger(job)
 699 }
 700
 701
 702 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 703     assert os instanceof String
 704     assert architecture instanceof String
 705
 706     def armArches = ['arm', 'armem', 'arm64']
 707
 708     if (!(architecture in armArches)) {
 709         assert options == null
 710         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 711
 712         return
 713     }
 714
 715     // This is an arm(64) job.
 716     //
 717     // There are several options.
 718     //
 719     // Windows_NT
 720     //
 721     // Arm32 (Build) -> latest-arm64
 722     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 723     // Arm32 (Test)  -> arm64-windows_nt
 724     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 725     //
 726     // Arm64 (Build) -> latest-arm64
 727     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 728     // Arm64 (Test)  -> arm64-windows_nt
 729     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 730     //
 731     // Ubuntu
 732     //
 733     // Arm32 emulator (Build, Test) -> arm-cross-latest
 734     //       |-> os == "Tizen" && (architecture == "armem")
 735     //
 736     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 737     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 738     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 739     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 740     // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
 741     //       |-> os == "Ubuntu" && (architecture == "arm")
 742     //
 743     // Arm64 (Build) -> arm64-cross-latest
 744     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
 745     // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
 746     //       |-> os != "Windows_NT" && architecture == "arm64"
 747     //
 748     // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
 749     // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
 750     //
 751     // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
 752     // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 753
 754     // This has to be a arm arch
 755     assert architecture in armArches
 756     if (os == "Windows_NT") {
 757         // arm32/arm64 Windows jobs share the same machines for now
 758         def isBuild = options['use_arm64_build_machine'] == true
 759
 760         if (isBuild == true) {
 761             job.with {
 762                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 763             }
 764         } else {
 765             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 766         }
 767     } else {
 768         assert os != 'Windows_NT'
 769
 770         if (architecture == 'armem') {
 771             // arm emulator (Tizen). Build and test on same machine,
 772             // using Docker.
 773             assert os == 'Tizen'
 774             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 775         }
 776         else {
 777             // arm/arm64 Ubuntu on hardware.
 778             assert architecture == 'arm' || architecture == 'arm64'
 779             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 780             def isBuild = (options != null) && (options['is_build_job'] == true)
 781             if (isFlow || isBuild) {
 782                 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 783                 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
 784                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 785             } else {
 786                 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
 787                 if (architecture == 'arm64') {
 788                     assert os == 'Ubuntu16.04'
 789                     job.with {
 790                         label('Ubuntu.1604.Arm64.Open')
 791                     }
 792                 }
 793                 else {
 794                     assert os == 'Ubuntu'
 795                     job.with {
 796                         label('ubuntu.1404.arm32.open')
 797                     }
 798                 }
 799             }
 800         }
 801     }
 802 }
 803
 804 // setJobMachineAffinity: compute the machine affinity options for a job,
 805 // then set the job with those affinity options.
 806 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 807 {
 808     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 809            (!isBuildJob && isTestJob  && !isFlowJob) ||
 810            (!isBuildJob && !isTestJob && isFlowJob)
 811
 812     def affinityOptions = null
 813     def affinityArchitecture = architecture
 814
 815     if (os == "Windows_NT") {
 816         if (architecture in Constants.armWindowsCrossArchitectureList) {
 817             if (isBuildJob) {
 818                 affinityOptions = [ "use_arm64_build_machine" : true ]
 819             } else if (isTestJob) {
 820                 affinityOptions = [ "use_arm64_build_machine" : false ]
 821             } else if (isFlowJob) {
 822                 // For the flow jobs set the machine affinity as x64
 823                 affinityArchitecture = 'x64'
 824             }
 825         }
 826     }
 827     else {
 828         if ((architecture == 'arm64') || (architecture == 'arm')) {
 829             if (isBuildJob) {
 830                 affinityOptions = ['is_build_job': true]
 831             } else if (isFlowJob) {
 832                 affinityOptions = ['is_flow_job': true]
 833             }
 834         }
 835     }
 836
 837     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 838 }
 839
 840 def static isGCStressRelatedTesting(def scenario) {
 841     // The 'r2r_gcstress15' scenario is a basic scenario.
 842     // Detect it and make it a GCStress related.
 843     if (scenario == 'r2r_gcstress15')
 844     {
 845         return true;
 846     }
 847
 848     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 849     def scenarioName = scenario.toLowerCase()
 850     def isGCStressTesting = false
 851     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 852         if (k in gcStressTestEnvVars) {
 853             isGCStressTesting = true;
 854         }
 855     }
 856     return isGCStressTesting
 857 }
 858
 859 def static isCoreFxScenario(def scenario) {
 860     def corefx_prefix = 'corefx_'
 861     if (scenario.length() < corefx_prefix.length()) {
 862         return false
 863     }
 864     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 865 }
 866
 867 def static isR2RBaselineScenario(def scenario) {
 868     return (scenario == 'r2r')
 869 }
 870
 871 def static isR2RStressScenario(def scenario) {
 872     return Constants.r2rStressScenarios.containsKey(scenario)
 873 }
 874
 875 def static isR2RScenario(def scenario) {
 876     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 877 }
 878
 879 def static isJitStressScenario(def scenario) {
 880     return Constants.jitStressModeScenarios.containsKey(scenario)
 881 }
 882
 883 def static isLongGc(def scenario) {
 884     return (scenario == 'longgc' || scenario == 'gcsimulator')
 885 }
 886
 887 def static isJitDiff(def scenario) {
 888     return (scenario == 'jitdiff')
 889 }
 890
 891 def static isGcReliabilityFramework(def scenario) {
 892     return (scenario == 'gc_reliability_framework')
 893 }
 894
 895 def static isArmWindowsScenario(def scenario) {
 896     return Constants.validArmWindowsScenarios.containsKey(scenario)
 897 }
 898
 899 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 900     if (isBuildOnly == true) {
 901         os = 'Windows_NT_BuildOnly'
 902     }
 903
 904     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 905     if (validOsPrTriggerArchConfigs != null) {
 906         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 907         if (validOsPrTriggerConfigs != null) {
 908             if (configuration in validOsPrTriggerConfigs) {
 909                 return true
 910             }
 911         }
 912     }
 913
 914     return false
 915 }
 916
 917 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 918 // scheduled with a default PR trigger despite the correlation being true at the moment.
 919 def static isInnerloopTestScenario(def scenario) {
 920     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 921 }
 922
 923 def static isCrossGenComparisonScenario(def scenario) {
 924     return (scenario == 'crossgen_comparison')
 925 }
 926
 927 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 928     assert isCrossGenComparisonScenario(scenario)
 929     return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
 930 }
 931
 932 def static getFxBranch(def branch) {
 933     def fxBranch = branch
 934     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 935     // branch even though CoreFX doesn't have such a branch.
 936     if (branch == 'dev/unix_test_workflow') {
 937         fxBranch = 'master'
 938     }
 939     return fxBranch
 940 }
 941
 942 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 943     // 2 hours (120 minutes) is the default timeout
 944     def timeout = 120
 945
 946     if (!isInnerloopTestScenario(scenario)) {
 947         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 948         timeout = 240
 949     }
 950
 951     if (!isBuildOnly) {
 952         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 953         if (isGCStressRelatedTesting(scenario)) {
 954             timeout = 4320
 955         }
 956         else if (isCoreFxScenario(scenario)) {
 957             timeout = 360
 958         }
 959         else if (isJitStressScenario(scenario)) {
 960             timeout = 300
 961         }
 962         else if (isR2RBaselineScenario(scenario)) {
 963             timeout = 240
 964         }
 965         else if (isLongGc(scenario)) {
 966             timeout = 1440
 967         }
 968         else if (isJitDiff(scenario)) {
 969             timeout = 240
 970         }
 971         else if (isGcReliabilityFramework(scenario)) {
 972             timeout = 1440
 973         }
 974         else if (architecture == 'armem' || architecture == 'arm64') {
 975             timeout = 240
 976         }
 977
 978         if (architecture == 'arm') {
 979             // ARM32 machines are particularly slow.
 980             timeout += 120
 981         }
 982     }
 983
 984     if (configuration == 'Debug') {
 985         // Debug runs can be very slow. Add an hour.
 986         timeout += 60
 987     }
 988
 989     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 990         // AltJit runs compile all methods twice.
 991         timeout *= 2
 992     }
 993
 994     // If we've changed the timeout from the default, set it in the job.
 995
 996     if (timeout != 120) {
 997         Utilities.setJobTimeout(newJob, timeout)
 998     }
 999 }
1000
1001 def static getJobFolder(def scenario) {
1002     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1003         return 'jitstress'
1004     }
1005     if (scenario == 'illink') {
1006         return 'illink'
1007     }
1008     return ''
1009 }
1010
1011 def static getStressModeDisplayName(def scenario) {
1012     def displayStr = ''
1013     Constants.jitStressModeScenarios[scenario].each{ k, v ->
1014         def prefixLength = 'COMPlus_'.length()
1015         if (k.length() >= prefixLength) {
1016             def modeName = k.substring(prefixLength, k.length())
1017             if (displayStr != '') {
1018                 // Separate multiple variables with a space.
1019                 displayStr += ' '
1020             }
1021             displayStr += modeName + '=' + v
1022         }
1023     }
1024
1025     if (isCoreFxScenario(scenario)) {
1026         displayStr = ('CoreFx ' + displayStr).trim()
1027     }
1028
1029     return displayStr
1030 }
1031
1032 def static getR2RDisplayName(def scenario) {
1033     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1034     def displayStr = scenario
1035     def prefixLength = 'r2r_'.length()
1036     if (displayStr.length() >= prefixLength) {
1037         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1038     } else if (scenario == 'r2r') {
1039         displayStr = "R2R"
1040     }
1041     return displayStr
1042 }
1043
1044 def static getScenarioDisplayString(def scenario) {
1045     switch (scenario) {
1046         case 'innerloop':
1047             return "Innerloop Build and Test"
1048
1049         case 'no_tiered_compilation_innerloop':
1050             def displayStr = getStressModeDisplayName(scenario)
1051             return "Innerloop Build and Test (Jit - ${displayStr})"
1052
1053         case 'corefx_innerloop':
1054             return "CoreFX Tests"
1055
1056         case 'normal':
1057             return "Build and Test"
1058
1059         case 'jitdiff':
1060             return "Jit Diff Build and Test"
1061
1062         case 'ilrt':
1063             return "IL RoundTrip Build and Test"
1064
1065         case 'longgc':
1066             return "Long-Running GC Build & Test"
1067
1068         case 'gcsimulator':
1069             return "GC Simulator"
1070
1071         case 'standalone_gc':
1072             return "Standalone GC"
1073
1074         case 'gc_reliability_framework':
1075             return "GC Reliability Framework"
1076
1077         case 'illink':
1078             return "via ILLink"
1079
1080         default:
1081             if (isJitStressScenario(scenario)) {
1082                 def displayStr = getStressModeDisplayName(scenario)
1083                 return "Build and Test (Jit - ${displayStr})"
1084             }
1085             else if (isR2RScenario(scenario)) {
1086                 def displayStr = getR2RDisplayName(scenario)
1087                 return "${displayStr} Build and Test"
1088             }
1089             else {
1090                 return "${scenario}"
1091             }
1092             break
1093     }
1094
1095     println("Unknown scenario: ${scenario}");
1096     assert false
1097 }
1098
1099 //
1100 // Functions to create an environment script.
1101 //      envScriptCreate -- initialize the script (call first)
1102 //      envScriptFinalize -- finalize the script (call last)
1103 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1104 //      envScriptAppendExistingScript -- append an existing script to the generated script
1105 //
1106 // Each script returns a string of commands. Concatenate all the strings together before
1107 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1108 //
1109
1110 // Initialize the environment setting script.
1111 def static envScriptCreate(def os, def stepScriptLocation) {
1112     def stepScript = ''
1113     if (os == 'Windows_NT') {
1114         stepScript += "echo Creating TestEnv script\r\n"
1115         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1116
1117         // Create at least an empty script.
1118         stepScript += "echo. > ${stepScriptLocation}\r\n"
1119     }
1120     else {
1121         stepScript += "echo Creating environment setting script\n"
1122         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1123     }
1124
1125     return stepScript
1126 }
1127
1128 // Generates the string for setting stress mode variables.
1129 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1130     def stepScript = ''
1131     if (os == 'Windows_NT') {
1132         stressModeVars.each{ k, v ->
1133             // Write out what we are writing to the script file
1134             stepScript += "echo Setting ${k}=${v}\r\n"
1135             // Write out the set itself to the script file`
1136             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1137         }
1138     }
1139     else {
1140         stressModeVars.each{ k, v ->
1141             // Write out what we are writing to the script file
1142             stepScript += "echo Setting ${k}=${v}\n"
1143             // Write out the set itself to the script file`
1144             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1145         }
1146     }
1147
1148     return stepScript
1149 }
1150
1151 // Append an existing script to an environment script.
1152 // Returns string of commands to do this.
1153 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1154     assert (os == 'Windows_NT')
1155     def stepScript = ''
1156
1157     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1158     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1159
1160     return stepScript
1161 }
1162
1163 // Finalize an environment setting script.
1164 // Returns string of commands to do this.
1165 def static envScriptFinalize(def os, def stepScriptLocation) {
1166     def stepScript = ''
1167
1168     if (os == 'Windows_NT') {
1169         // Display the resulting script. This is useful when looking at the output log file.
1170         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1171         stepScript += "type ${stepScriptLocation}\r\n"
1172     }
1173     else {
1174         stepScript += "chmod +x ${stepScriptLocation}\n"
1175     }
1176
1177     return stepScript
1178 }
1179
1180 def static isNeedDocker(def architecture, def os, def isBuild) {
1181     if (isBuild) {
1182         if (architecture == 'x86' && os == 'Ubuntu') {
1183             return true
1184         }
1185         else if (architecture == 'armem') {
1186             return true
1187         }
1188         else if (architecture == 'arm') {
1189             if (os == 'Ubuntu') {
1190                 return true
1191             }
1192         }
1193         else if (architecture == 'arm64') {
1194             if (os == 'Ubuntu16.04') {
1195                 return true
1196             }
1197         }
1198     }
1199     else {
1200         if (architecture == 'x86' && os == 'Ubuntu') {
1201             return true
1202         }
1203     }
1204     return false
1205 }
1206
1207 def static getDockerImageName(def architecture, def os, def isBuild) {
1208     // We must change some docker private images to official later
1209     if (isBuild) {
1210         if (architecture == 'x86' && os == 'Ubuntu') {
1211             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1212         }
1213         else if (architecture == 'armem') {
1214             if (os == 'Tizen') {
1215                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1216             }
1217         }
1218         else if (architecture == 'arm') {
1219             if (os == 'Ubuntu') {
1220                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1221             }
1222         }
1223         else if (architecture == 'arm64') {
1224             if (os == 'Ubuntu16.04') {
1225                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1226             }
1227         }
1228     }
1229     else {
1230         if (architecture == 'x86' && os == 'Ubuntu') {
1231             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1232         }
1233     }
1234     println("Unknown architecture to use docker: ${architecture} ${os}");
1235     assert false
1236 }
1237
1238
1239 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1240 // and only allowing using this hardware in some specific branches.
1241 def static jobRequiresLimitedHardware(def architecture, def os) {
1242     if (architecture == 'arm') {
1243         // arm Windows and Linux hardware is limited.
1244         return true
1245     }
1246     else if (architecture == 'arm64') {
1247         // arm64 Windows and Linux hardware is limited.
1248         return true
1249     }
1250     else {
1251         return false
1252     }
1253 }
1254
1255 // Calculates the name of the build job based on some typical parameters.
1256 //
1257 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1258     // If the architecture is x64, do not add that info into the build name.
1259     // Need to change around some systems and other builds to pick up the right builds
1260     // to do that.
1261
1262     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1263     if (isBuildOnly) {
1264         suffix += '_bld'
1265     }
1266     def baseName = ''
1267     switch (architecture) {
1268         case 'x64':
1269             if (scenario == 'normal') {
1270                 // For now we leave x64 off of the name for compatibility with other jobs
1271                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1272             }
1273             else if (scenario == 'formatting') {
1274                 // we don't care about the configuration for the formatting job. It runs all configs
1275                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1276             }
1277             else {
1278                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1279             }
1280             break
1281         case 'armem':
1282             // These are cross builds
1283             assert os == 'Tizen'
1284             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1285             break
1286         case 'arm':
1287         case 'arm64':
1288             // These are cross builds
1289             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1290             break
1291         case 'x86':
1292         case 'x86_arm_altjit':
1293         case 'x64_arm64_altjit':
1294             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1295             break
1296         default:
1297             println("Unknown architecture: ${architecture}");
1298             assert false
1299             break
1300     }
1301
1302     return baseName + suffix
1303 }
1304
1305 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1306     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1307
1308     // Limited hardware is restricted for non-PR triggers to certain branches.
1309     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1310         return
1311     }
1312
1313     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1314     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1315     if (architecture == 'x86' && os == 'Ubuntu') {
1316         return
1317     }
1318
1319     // Check scenario.
1320     switch (scenario) {
1321         case 'innerloop':
1322         case 'no_tiered_compilation_innerloop':
1323             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1324             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1325             // We should never get here (in the "innerloop cases) for non-PR jobs, except for this TEMPORARY exception.
1326             // Only trigger the flow job, not the build job.
1327             assert (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked'))
1328             if (isFlowJob) {
1329                 addGithubPushTriggerHelper(job)
1330             }
1331             break
1332
1333         case 'crossgen_comparison':
1334             break
1335         case 'normal':
1336             switch (architecture) {
1337                 case 'x64':
1338                 case 'x86':
1339                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1340                         addPeriodicTriggerHelper(job, '@daily')
1341                     }
1342                     else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1343                         addGithubPushTriggerHelper(job)
1344                     }
1345                     break
1346                 case 'arm64':
1347                     if (os == 'Windows_NT') {
1348                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1349                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1350                             // Do the builds daily.
1351                             addPeriodicTriggerHelper(job, '@daily')
1352                         }
1353                     }
1354                     else {
1355                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1356                         if (isFlowJob) {
1357                             addPeriodicTriggerHelper(job, '@daily')
1358                         }
1359                     }
1360                     break
1361                 case 'arm':
1362                     if (os == 'Windows_NT') {
1363                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1364                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1365                             // Do the builds daily.
1366                             addPeriodicTriggerHelper(job, '@daily')
1367                         }
1368                     }
1369                     else {
1370                         assert os == 'Ubuntu'
1371                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1372                         if (isFlowJob) {
1373                             // Currently no push triggers, with limited arm Linux hardware.
1374                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1375                             addPeriodicTriggerHelper(job, '@daily')
1376                         }
1377                     }
1378                     break
1379                 case 'armem':
1380                     addGithubPushTriggerHelper(job)
1381                     break
1382                 case 'x86_arm_altjit':
1383                 case 'x64_arm64_altjit':
1384                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1385                     if (configuration == 'Checked') {
1386                         addGithubPushTriggerHelper(job)
1387                     }
1388                     break
1389                 default:
1390                     println("Unknown architecture: ${architecture}");
1391                     assert false
1392                     break
1393             }
1394             break
1395         case 'r2r':
1396             assert !(os in bidailyCrossList)
1397             // r2r gets a push trigger for checked/release
1398             if (configuration == 'Checked' || configuration == 'Release') {
1399                 if (architecture == 'x64' && os != 'OSX10.12') {
1400                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1401                     if (isFlowJob || os == 'Windows_NT') {
1402                         addGithubPushTriggerHelper(job)
1403                     }
1404                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1405                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1406                     if (isFlowJob) {
1407                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1408                     }
1409                 }
1410                 // For x86, only add per-commit jobs for Windows
1411                 else if (architecture == 'x86') {
1412                     if (os == 'Windows_NT') {
1413                         addGithubPushTriggerHelper(job)
1414                     }
1415                 }
1416                 // arm r2r jobs should only run weekly.
1417                 else if (architecture == 'arm') {
1418                     if (isFlowJob) {
1419                         addPeriodicTriggerHelper(job, '@weekly')
1420                     }
1421                 }
1422                 // arm64 r2r jobs should only run weekly.
1423                 else if (architecture == 'arm64') {
1424                     if (isFlowJob) {
1425                         addPeriodicTriggerHelper(job, '@weekly')
1426                     }
1427                 }
1428             }
1429             break
1430         case 'r2r_jitstress1':
1431         case 'r2r_jitstress2':
1432         case 'r2r_jitstress1_tiered':
1433         case 'r2r_jitstress2_tiered':
1434         case 'r2r_jitstressregs1':
1435         case 'r2r_jitstressregs2':
1436         case 'r2r_jitstressregs3':
1437         case 'r2r_jitstressregs4':
1438         case 'r2r_jitstressregs8':
1439         case 'r2r_jitstressregs0x10':
1440         case 'r2r_jitstressregs0x80':
1441         case 'r2r_jitstressregs0x1000':
1442         case 'r2r_jitminopts':
1443         case 'r2r_jitforcerelocs':
1444         case 'r2r_gcstress15':
1445         case 'r2r_no_tiered_compilation':
1446             assert !(os in bidailyCrossList)
1447
1448             // GCStress=C is currently not supported on OS X
1449             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1450                 break
1451             }
1452
1453             if (configuration == 'Checked' || configuration == 'Release') {
1454                 if (architecture == 'x64') {
1455                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1456                     if (isFlowJob || os == 'Windows_NT') {
1457                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1458                     }
1459                 }
1460                 // For x86, only add periodic jobs for Windows
1461                 else if (architecture == 'x86') {
1462                     if (os == 'Windows_NT') {
1463                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1464                     }
1465                 }
1466                 else if (architecture == 'arm') {
1467                     if (isFlowJob) {
1468                         addPeriodicTriggerHelper(job, '@weekly')
1469                     }
1470                 }
1471                 else if (architecture == 'arm64') {
1472                     if (isFlowJob) {
1473                         addPeriodicTriggerHelper(job, '@weekly')
1474                     }
1475                 }
1476             }
1477             break
1478         case 'longgc':
1479             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1480             assert configuration == 'Release'
1481             assert architecture == 'x64'
1482             addPeriodicTriggerHelper(job, '@daily')
1483             // TODO: Add once external email sending is available again
1484             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1485             break
1486         case 'gcsimulator':
1487             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1488             assert configuration == 'Release'
1489             assert architecture == 'x64'
1490             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1491             // TODO: Add once external email sending is available again
1492             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1493             break
1494         case 'standalone_gc':
1495             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1496             assert (configuration == 'Release' || configuration == 'Checked')
1497             // TODO: Add once external email sending is available again
1498             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1499             addPeriodicTriggerHelper(job, '@daily')
1500             break
1501         case 'gc_reliability_framework':
1502             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1503             assert (configuration == 'Release' || configuration == 'Checked')
1504             // Only triggered by phrase.
1505             break
1506         case 'ilrt':
1507             assert !(os in bidailyCrossList)
1508             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1509             if (architecture == 'x64' && configuration == 'Release') {
1510                 if (isFlowJob || os == 'Windows_NT') {
1511                     addPeriodicTriggerHelper(job, '@daily')
1512                 }
1513             }
1514             break
1515         case 'jitdiff':
1516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1517             assert configuration == 'Checked'
1518             assert (architecture == 'x64' || architecture == 'x86')
1519             addGithubPushTriggerHelper(job)
1520             break
1521         case 'formatting':
1522             assert (os == 'Windows_NT' || os == "Ubuntu")
1523             assert architecture == 'x64'
1524             addGithubPushTriggerHelper(job)
1525             break
1526         case 'jitstressregs1':
1527         case 'jitstressregs2':
1528         case 'jitstressregs3':
1529         case 'jitstressregs4':
1530         case 'jitstressregs8':
1531         case 'jitstressregs0x10':
1532         case 'jitstressregs0x80':
1533         case 'jitstressregs0x1000':
1534         case 'minopts':
1535         case 'tieredcompilation':
1536         case 'no_tiered_compilation':
1537         case 'forcerelocs':
1538         case 'jitstress1':
1539         case 'jitstress2':
1540         case 'jitstress1_tiered':
1541         case 'jitstress2_tiered':
1542         case 'jitstress2_jitstressregs1':
1543         case 'jitstress2_jitstressregs2':
1544         case 'jitstress2_jitstressregs3':
1545         case 'jitstress2_jitstressregs4':
1546         case 'jitstress2_jitstressregs8':
1547         case 'jitstress2_jitstressregs0x10':
1548         case 'jitstress2_jitstressregs0x80':
1549         case 'jitstress2_jitstressregs0x1000':
1550         case 'tailcallstress':
1551         case 'jitsse2only':
1552         case 'jitnosimd':
1553         case 'jitnox86hwintrinsic':
1554         case 'jitincompletehwintrinsic':
1555         case 'jitx86hwintrinsicnoavx':
1556         case 'jitx86hwintrinsicnoavx2':
1557         case 'jitx86hwintrinsicnosimd':
1558         case 'corefx_baseline':
1559         case 'corefx_minopts':
1560         case 'corefx_tieredcompilation':
1561         case 'corefx_jitstress1':
1562         case 'corefx_jitstress2':
1563         case 'corefx_jitstressregs1':
1564         case 'corefx_jitstressregs2':
1565         case 'corefx_jitstressregs3':
1566         case 'corefx_jitstressregs4':
1567         case 'corefx_jitstressregs8':
1568         case 'corefx_jitstressregs0x10':
1569         case 'corefx_jitstressregs0x80':
1570         case 'corefx_jitstressregs0x1000':
1571         case 'zapdisable':
1572             if (os == 'CentOS7.1') {
1573                 break
1574             }
1575             if (os in bidailyCrossList) {
1576                 break
1577             }
1578             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1579             // build for the flow job. We don't need cron jobs for these. Note that the
1580             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1581             // function very early, so only non-Windows gets here.
1582             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1583                 break
1584             }
1585             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1586                 break
1587             }
1588             if (jobRequiresLimitedHardware(architecture, os)) {
1589                 if ((architecture == 'arm64') && (os == 'Ubuntu16.04') && !isCoreFxScenario(scenario)) {
1590                     // These jobs are very fast on Linux/arm64 hardware, so run them daily.
1591                     // TODO: When the corefx jobs are made to run in parallel, run those
1592                     // jobs daily as well.
1593                     addPeriodicTriggerHelper(job, '@daily')
1594                 }
1595                 else {
1596                     addPeriodicTriggerHelper(job, '@weekly')
1597                 }
1598             }
1599             else {
1600                 addPeriodicTriggerHelper(job, '@daily')
1601             }
1602             break
1603         case 'heapverify1':
1604         case 'gcstress0x3':
1605             if (os == 'CentOS7.1') {
1606                 break
1607             }
1608             if (os in bidailyCrossList) {
1609                 break
1610             }
1611             addPeriodicTriggerHelper(job, '@weekly')
1612             break
1613         case 'gcstress0xc':
1614         case 'gcstress0xc_zapdisable':
1615         case 'gcstress0xc_zapdisable_jitstress2':
1616         case 'gcstress0xc_zapdisable_heapverify1':
1617         case 'gcstress0xc_jitstress1':
1618         case 'gcstress0xc_jitstress2':
1619         case 'gcstress0xc_minopts_heapverify1':
1620             if (os == 'OSX10.12') {
1621                 // GCStress=C is currently not supported on OS X
1622                 break
1623             }
1624             if (os == 'CentOS7.1') {
1625                 break
1626             }
1627             if (os in bidailyCrossList) {
1628                 break
1629             }
1630             addPeriodicTriggerHelper(job, '@weekly')
1631             break
1632
1633         case 'illink':
1634             // Testing on other operating systems TBD
1635             assert (os == 'Windows_NT' || os == 'Ubuntu')
1636             if (architecture == 'x64' || architecture == 'x86') {
1637                 if (configuration == 'Checked') {
1638                     addPeriodicTriggerHelper(job, '@daily')
1639                 }
1640             }
1641             break
1642
1643         default:
1644             println("Unknown scenario: ${scenario}");
1645             assert false
1646             break
1647     }
1648     return
1649 }
1650
1651 // **************************
1652 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1653 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1654 // builds will do a couple extra steps.
1655 // **************************
1656
1657 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1658 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1659 // it's just used for internal builds
1660 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1661 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1662     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1663
1664     if (isWindowsBuildOnlyJob) {
1665         return
1666     }
1667
1668     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1669     // Non pull request builds.
1670     if (!isPR) {
1671         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1672         return
1673     }
1674
1675     def arm64Users = [
1676         'adityamandaleeka',
1677         'AndyAyersMS',
1678         'briansull',
1679         'BruceForstall',
1680         'CarolEidt',
1681         'davidwrighton',
1682         'echesakovMSFT',
1683         'erozenfeld',
1684         'janvorli',
1685         'jashook',
1686         'pgodeq',
1687         'RussKeldorph',
1688         'sandreenko',
1689         'sdmaclea',
1690         'swaroop-sridhar',
1691         'jkotas',
1692         'markwilkie',
1693         'weshaggard'
1694     ]
1695
1696     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1697     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1698
1699     def contextString = ""
1700     def triggerString = ""
1701     def needsTrigger = true
1702     def isDefaultTrigger = false
1703     def isArm64PrivateJob = false
1704     def scenarioString = ""
1705
1706     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1707     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1708     // some jobs where the Debug configuration just does builds, no tests.
1709     //
1710     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1711     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1712     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1713     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1714
1715     if (architecture == 'armem') {
1716         assert os == 'Tizen'
1717         architecture = 'armel'
1718     }
1719
1720     switch (architecture) {
1721         case 'x64_arm64_altjit':
1722         case 'x86_arm_altjit':
1723             // TODO: for consistency, add "Build and Test" at end.
1724             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1725             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1726             break
1727
1728         case 'armel':
1729         case 'arm':
1730         case 'arm64':
1731             contextString = "${os} ${architecture} Cross ${configuration}"
1732             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1733
1734             if (scenario == 'innerloop') {
1735                 contextString += " Innerloop"
1736                 triggerString += "\\W+Innerloop"
1737             }
1738             else {
1739                 contextString += " ${scenario}"
1740                 triggerString += "\\W+${scenario}"
1741             }
1742
1743             if (configuration == 'Debug') {
1744                 contextString += " Build"
1745                 triggerString += "\\W+Build"
1746             }
1747             else {
1748                 contextString += " Build and Test"
1749                 triggerString += "\\W+Build and Test"
1750             }
1751
1752             triggerString += ".*"
1753             break
1754
1755         default:
1756             scenarioString = getScenarioDisplayString(scenario)
1757             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1758             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1759
1760             switch (scenario) {
1761                 case 'normal':
1762                     triggerString += "\\W+Build and Test.*"
1763                     break
1764
1765                 case 'corefx_innerloop': // maintain this asymmetry
1766                     triggerString += "\\W+CoreFX Tests.*"
1767                     break
1768
1769                 default:
1770                     triggerString += "\\W+${scenario}.*"
1771                     break
1772             }
1773
1774             triggerString += ".*"
1775             break
1776     }
1777
1778     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1779     // non-flow jobs that are only used as part of flow jobs.
1780
1781     switch (architecture) {
1782         case 'x64': // editor brace matching: {
1783             if (scenario == 'formatting') {
1784                 assert configuration == 'Checked'
1785                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1786                     isDefaultTrigger = true
1787                     contextString = "${os} ${architecture} Formatting"
1788                 }
1789                 break
1790             }
1791
1792             switch (os) {
1793                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1794                 case 'Debian8.4':
1795                 case 'RHEL7.2':
1796                     if (scenario == 'innerloop') {
1797                         assert !isFlowJob
1798                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1799                         isDefaultTrigger = true
1800                         break
1801                     }
1802
1803                     // fall through
1804
1805                 case 'Fedora24':
1806                 case 'Ubuntu16.04':
1807                 case 'Ubuntu16.10':
1808                     assert !isFlowJob
1809                     assert scenario != 'innerloop'
1810                     contextString = "${os} ${architecture} ${configuration} Build"
1811                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1812                     break
1813
1814                 case 'Ubuntu':
1815                     if (scenario == 'illink') {
1816                         break
1817                     }
1818                     else if (scenario == 'corefx_innerloop') {
1819                         if (configuration == 'Checked') {
1820                             isDefaultTrigger = true
1821                         }
1822                         break
1823                     }
1824
1825                     // fall through
1826
1827                 case 'OSX10.12':
1828                     // Triggers on the non-flow jobs aren't necessary here
1829                     // Corefx testing uses non-flow jobs.
1830                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1831                         needsTrigger = false
1832                         break
1833                     }
1834                     switch (scenario) {
1835                         case 'innerloop':
1836                             isDefaultTrigger = true
1837                             break
1838
1839                         case 'no_tiered_compilation_innerloop':
1840                             if (os == 'Ubuntu') {
1841                                 isDefaultTrigger = true
1842                             }
1843                             break
1844
1845                         default:
1846                             break
1847                     }
1848                     break
1849
1850                 case 'CentOS7.1':
1851                     switch (scenario) {
1852                         case 'innerloop':
1853                             // CentOS uses checked for default PR tests while debug is build only
1854                             if (configuration == 'Debug') {
1855                                 isDefaultTrigger = true
1856                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1857                                 break
1858                             }
1859
1860                             // Make sure this is a flow job to get build and test.
1861                             if (!isFlowJob) {
1862                                 needsTrigger = false
1863                                 break
1864                             }
1865
1866                             if (configuration == 'Checked') {
1867                                 assert job.name.contains("flow")
1868                                 isDefaultTrigger = true
1869                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1870                             }
1871                             break
1872
1873                         case 'normal':
1874                             // Make sure this is a flow job to get build and test.
1875                             if (!isFlowJob) {
1876                                 needsTrigger = false
1877                                 break
1878                             }
1879                             break
1880
1881                         default:
1882                             break
1883                     }
1884                     break
1885
1886                 case 'Windows_NT':
1887                     switch (scenario) {
1888                         case 'innerloop':
1889                         case 'no_tiered_compilation_innerloop':
1890                             isDefaultTrigger = true
1891                             break
1892
1893                         case 'corefx_innerloop':
1894                             if (configuration == 'Checked' || configuration == 'Release') {
1895                                 isDefaultTrigger = true
1896                             }
1897                             break
1898
1899                         default:
1900                             break
1901                     }
1902                     break
1903
1904                 default:
1905                     println("Unknown os: ${os}");
1906                     assert false
1907                     break
1908
1909             } // switch (os)
1910
1911             break
1912         // editor brace matching: }
1913
1914         case 'armel': // editor brace matching: {
1915             job.with {
1916                 publishers {
1917                     azureVMAgentPostBuildAction {
1918                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1919                     }
1920                 }
1921             }
1922
1923             switch (os) {
1924                 case 'Tizen':
1925                     if (scenario == 'innerloop') {
1926                         if (configuration == 'Checked') {
1927                             isDefaultTrigger = true
1928                         }
1929                     }
1930                     break
1931             }
1932
1933             break
1934         // editor brace matching: }
1935
1936         case 'arm':
1937         case 'arm64': // editor brace matching: {
1938
1939             switch (os) {
1940                 case 'Ubuntu':
1941                 case 'Ubuntu16.04':
1942
1943                     // Triggers on the non-flow jobs aren't necessary
1944                     if (!isFlowJob) {
1945                         needsTrigger = false
1946                         break
1947                     }
1948
1949                     switch (scenario) {
1950                         case 'innerloop':
1951                         case 'no_tiered_compilation_innerloop':
1952                             if (configuration == 'Checked') {
1953                                 // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1954                                 //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1955                                 if (architecture == 'arm64') {
1956                                     break
1957                                 }
1958                                 isDefaultTrigger = true
1959                             }
1960                             break
1961                     }
1962                     break
1963
1964                 case 'Windows_NT':
1965                     assert isArmWindowsScenario(scenario)
1966
1967                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1968                     // the non-flow Build job. All others need a trigger on the flow job.
1969                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1970                     if (isFlowJob != needsFlowJobTrigger) {
1971                         needsTrigger = false
1972                         break
1973                     }
1974
1975                     switch (scenario) {
1976                         case 'innerloop':
1977                             if (configuration == 'Debug') {
1978                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1979                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1980                                 // People in arm64Users will get both this and the Checked Build and Test job.
1981                                 isDefaultTrigger = true
1982                             } else if (configuration == 'Checked') {
1983                                 isDefaultTrigger = true
1984                                 isArm64PrivateJob = true
1985                             }
1986                             break
1987                         default:
1988                             isArm64PrivateJob = true
1989                             break
1990                     }
1991                     break
1992                 default:
1993                     println("NYI os: ${os}");
1994                     assert false
1995                     break
1996             }
1997             break
1998
1999         // editor brace matching: }
2000         case 'x86': // editor brace matching: {
2001             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2002             if (os == 'Ubuntu') {
2003                 // Triggers on the non-flow jobs aren't necessary here
2004                 if (!isFlowJob) {
2005                     needsTrigger = false
2006                     break
2007                 }
2008
2009                 // on-demand only for ubuntu x86
2010                 contextString = "${os} ${architecture} ${configuration} Build"
2011                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2012                 break
2013             }
2014             switch (scenario) {
2015                 case 'innerloop':
2016                 case 'no_tiered_compilation_innerloop':
2017                     isDefaultTrigger = true
2018                     break
2019                 default:
2020                     break
2021             }
2022             break
2023
2024         // editor brace matching: }
2025         case 'x64_arm64_altjit':
2026         case 'x86_arm_altjit':
2027             // Everything default
2028             break
2029
2030         default:
2031             println("Unknown architecture: ${architecture}");
2032             assert false
2033             break
2034     }
2035
2036     if (needsTrigger) {
2037         if (isArm64PrivateJob) {
2038             if (isDefaultTrigger) {
2039                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2040             }
2041             else {
2042                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2043             }
2044         }
2045         else {
2046             if (isDefaultTrigger) {
2047                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2048             }
2049             else {
2050                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2051             }
2052         }
2053     }
2054 }
2055
2056 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2057     def buildCommands = []
2058     def osGroup = getOSGroup(os)
2059     def lowerConfiguration = configuration.toLowerCase()
2060
2061     def priority = '1'
2062     if (isInnerloopTestScenario(scenario)) {
2063         priority = '0'
2064     }
2065
2066     def doCoreFxTesting = isCoreFxScenario(scenario)
2067
2068     // Calculate the build steps, archival, and xunit results
2069     switch (os) {
2070         case 'Windows_NT': // editor brace matching: {
2071             switch (architecture) {
2072                 case 'x64':
2073                 case 'x86':
2074                 case 'x86_arm_altjit':
2075                 case 'x64_arm64_altjit':
2076                     def arch = architecture
2077                     def buildOpts = ''
2078                     if (architecture == 'x86_arm_altjit') {
2079                         arch = 'x86'
2080                     }
2081                     else if (architecture == 'x64_arm64_altjit') {
2082                         arch = 'x64'
2083                     }
2084
2085                     if (scenario == 'formatting') {
2086                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2087                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2088                         break
2089                     }
2090
2091                     if (scenario == 'illink') {
2092                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2093                     }
2094
2095                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2096                     if ((lowerConfiguration == 'release') &&
2097                         (scenario in Constants.basicScenarios) &&
2098                         (architecture != 'x86_arm_altjit') &&
2099                         (architecture != 'x64_arm64_altjit')) {
2100
2101                         buildOpts += ' -enforcepgo'
2102                     }
2103
2104                     if (doCoreFxTesting) {
2105                         buildOpts += ' skiptests';
2106                     } else {
2107                         buildOpts += " -priority=${priority}"
2108                     }
2109
2110                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2111                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2112                     // 35 characters long.
2113
2114                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2115
2116                     if (!isBuildOnly) {
2117                         def runtestArguments = ''
2118                         def testOpts = 'collectdumps'
2119
2120                         if (isR2RScenario(scenario)) {
2121
2122                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2123                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2124                             // to cause the tests to be crossgen'ed.
2125
2126                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2127                                 testOpts += ' crossgenaltjit protononjit.dll'
2128                             } else {
2129                                 testOpts += ' crossgen'
2130                             }
2131
2132                             testOpts += ' runcrossgentests'
2133                         }
2134                         else if (scenario == 'jitdiff') {
2135                             testOpts += ' jitdisasm crossgen'
2136                         }
2137                         else if (scenario == 'ilrt') {
2138                             testOpts += ' ilasmroundtrip'
2139                         }
2140                         else if (isLongGc(scenario)) {
2141                             testOpts += " ${scenario} sequential"
2142                         }
2143                         else if (scenario == 'standalone_gc') {
2144                             testOpts += ' gcname clrgc.dll'
2145                         }
2146                         else if (scenario == 'illink') {
2147                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2148                         }
2149
2150                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2151                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2152                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2153                         // which is read by the xunit harness.)
2154                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2155                         {
2156                             def timeout = 1800000
2157                             testOpts += " timeout ${timeout}"
2158                         }
2159
2160                         // If we are running a stress mode, we should write out the set of key
2161                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2162
2163                         def envScriptPath = ''
2164                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2165                             def buildCommandsStr = ''
2166                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2167                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2168
2169                             if (isJitStressScenario(scenario)) {
2170                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2171                             }
2172                             else if (isR2RStressScenario(scenario)) {
2173                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2174                             }
2175
2176                             if (architecture == 'x86_arm_altjit') {
2177                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2178                             }
2179                             else if (architecture == 'x64_arm64_altjit') {
2180                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2181                             }
2182
2183                             envScriptFinalize(os, envScriptPath)
2184
2185                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2186                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2187                             // into a single script.
2188                             buildCommands += buildCommandsStr
2189                         }
2190                         else if (architecture == 'x86_arm_altjit') {
2191                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2192                         }
2193                         else if (architecture == 'x64_arm64_altjit') {
2194                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2195                         }
2196                         if (envScriptPath != '') {
2197                             testOpts += " TestEnv ${envScriptPath}"
2198                         }
2199
2200                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2201
2202                         if (doCoreFxTesting) {
2203                             if (scenario == 'corefx_innerloop') {
2204                                 // Create CORE_ROOT and testhost
2205                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2206                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2207
2208                                 // Archive and process (only) the test results
2209                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2210                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2211                             }
2212                             else {
2213                                 def workspaceRelativeFxRoot = "_/fx"
2214                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2215                                 def fxBranch = getFxBranch(branch)
2216
2217                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2218
2219                                 // Archive and process (only) the test results
2220                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2221                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2222
2223                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2224                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2225                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2226                             }
2227                         }
2228                         else if (isGcReliabilityFramework(scenario)) {
2229                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2230                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2231                         }
2232                         else {
2233                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2234                         }
2235                     } // end if (!isBuildOnly)
2236
2237                     if (!doCoreFxTesting) {
2238                         // Run the rest of the build
2239                         // Build the mscorlib for the other OS's
2240                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2241                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2242
2243                         if (arch == 'x64') {
2244                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2245                         }
2246
2247                         // Zip up the tests directory so that we don't use so much space/time copying
2248                         // 10s of thousands of files around.
2249                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2250
2251                         if (!isJitStressScenario(scenario)) {
2252                             // For Windows, pull full test results and test drops for x86/x64.
2253                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2254                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2255                         }
2256
2257                         if (scenario == 'jitdiff') {
2258                             // retrieve jit-dasm output for base commit, and run jit-diff
2259                             if (!isBuildOnly) {
2260                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2261                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2262                             }
2263                         }
2264
2265                         if (!isBuildOnly) {
2266                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2267                         }
2268                     }
2269                     break
2270                 case 'arm':
2271                 case 'arm64':
2272                     assert isArmWindowsScenario(scenario)
2273
2274                     def buildOpts = ''
2275
2276                     if (doCoreFxTesting) {
2277                         buildOpts += ' skiptests'
2278                     } else {
2279                         buildOpts += " -priority=${priority}"
2280                     }
2281
2282                     // This is now a build only job. Do not run tests. Use the flow job.
2283                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2284
2285                     if (doCoreFxTesting) {
2286                         assert isBuildOnly
2287
2288                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2289                         def envScriptPath = ''
2290                         def buildCommandsStr = ''
2291                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2292                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2293                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2294                         envScriptFinalize(os, envScriptPath)
2295                         buildCommands += buildCommandsStr
2296
2297                         def workspaceRelativeFxRootLinux = "_/fx"
2298                         def workspaceRelativeFxRootWin = "_\\fx"
2299                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2300                         def fxBranch = getFxBranch(branch)
2301
2302                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2303
2304                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2305                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2306                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2307
2308                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2309                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2310                     } else {
2311                         // Zip up the tests directory so that we don't use so much space/time copying
2312                         // 10s of thousands of files around.
2313                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2314
2315                         // Add archival.
2316                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2317                     }
2318                     break
2319                 default:
2320                     println("Unknown architecture: ${architecture}");
2321                     assert false
2322                     break
2323             }
2324             break
2325         // end case 'Windows_NT'; editor brace matching: }
2326         case 'Ubuntu':
2327         case 'Ubuntu16.04':
2328         case 'Ubuntu16.10':
2329         case 'Debian8.4':
2330         case 'OSX10.12':
2331         case 'CentOS7.1':
2332         case 'RHEL7.2':
2333         case 'Tizen':
2334         case 'Fedora24': // editor brace matching: {
2335             switch (architecture) {
2336                 case 'x64':
2337                 case 'x86':
2338                     if (architecture == 'x86' && os == 'Ubuntu') {
2339                         // build and PAL test
2340                         def dockerImage = getDockerImageName(architecture, os, true)
2341                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2342                         dockerImage = getDockerImageName(architecture, os, false)
2343                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2344                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2345                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2346                         break
2347                     }
2348
2349                     if (scenario == 'formatting') {
2350                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2351                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2352                         break
2353                     }
2354
2355                     if (scenario == 'illink') {
2356                         assert(os == 'Ubuntu')
2357                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2358                     }
2359
2360                     if (!doCoreFxTesting) {
2361                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2362                         // only on supported OS platforms.
2363                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2364                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2365
2366                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2367                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2368
2369                         // Basic archiving of the build
2370                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2371                         // And pal tests
2372                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2373                     }
2374                     else {
2375                         if (scenario == 'corefx_innerloop') {
2376                             assert os == 'Ubuntu' || 'OSX10.12'
2377                             assert architecture == 'x64'
2378
2379                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2380                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2381                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2382
2383                             break
2384                             // Archive and process (only) the test results
2385                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2386                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2387                         }
2388                         else {
2389                             // Corefx stress testing
2390                             assert os == 'Ubuntu'
2391                             assert architecture == 'x64'
2392                             assert lowerConfiguration == 'checked'
2393                             assert isJitStressScenario(scenario)
2394
2395                             // Build coreclr
2396                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2397
2398                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2399
2400                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2401                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2402                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2403                             buildCommands += envScriptCmds
2404
2405                             // Build and text corefx
2406                             def workspaceRelativeFxRoot = "_/fx"
2407                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2408                             def fxBranch = getFxBranch(branch)
2409
2410                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2411
2412                             // Archive and process (only) the test results
2413                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2414                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2415                         }
2416                     }
2417                     break
2418                 case 'armem':
2419                     // Emulator cross builds for ARM runs on Tizen currently
2420                     assert os == 'Tizen'
2421
2422                     def arm_abi = "armel"
2423                     def linuxCodeName = "tizen"
2424
2425                     // Unzip the Windows test binaries first. Exit with 0
2426                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2427
2428                     // Unpack the corefx binaries
2429                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2430                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2431
2432                     // Call the ARM CI script to cross build and test using docker
2433                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2434                     --mode=docker \\
2435                     --${arm_abi} \\
2436                     --linuxCodeName=${linuxCodeName} \\
2437                     --buildConfig=${lowerConfiguration} \\
2438                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2439                     --coreFxBinDir=./bin/CoreFxBinDir \\
2440                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2441
2442                     // Basic archiving of the build, no pal tests
2443                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2444                     break
2445                 case 'arm64':
2446                 case 'arm':
2447                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2448                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2449
2450                     // Add some useful information to the log file. Ignore return codes.
2451                     buildCommands += "uname -a || true"
2452
2453                     def additionalOpts = ""
2454                     if (architecture == 'arm') {
2455                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2456                     }
2457
2458                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2459                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2460                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2461                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2462                     // ZIP up the generated CoreFX runtime and tests.
2463
2464                     def dockerImage = getDockerImageName(architecture, os, true)
2465                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2466
2467                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2468
2469                     if (doCoreFxTesting) {
2470                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2471
2472                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2473                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2474                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2475                         buildCommands += envScriptCmds
2476
2477                         // Build and text corefx
2478                         def workspaceRelativeFxRootLinux = "_/fx"
2479                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2480                         def fxBranch = getFxBranch(branch)
2481
2482                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2483
2484                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2485                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2486                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2487
2488                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2489                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2490                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2491                     }
2492                     else if (isCrossGenComparisonScenario(scenario)) {
2493                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2494
2495                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2496                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2497                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2498                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2499                         def workspaceRelativeResultsDir = "_"
2500                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2501                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2502                         getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
2503                             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2504                             def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2505
2506                             buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2507                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2508                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2509                         } // crossArch
2510                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2511                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2512                     }
2513                     else if (architecture == 'arm') {
2514                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2515                         // download the appropriate CoreFX packages.
2516                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2517                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2518
2519                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2520
2521                         // ZIP up for the test job (created in the flow job code):
2522                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2523                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2524                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2525                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2526
2527                         // These commands are assumed to be run from the root of the workspace.
2528                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2529                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2530
2531                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2532                     }
2533                     else {
2534                         assert architecture == 'arm64'
2535
2536                         // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2537                         // Linux/arm64 does not use Windows-built tests.
2538
2539                         def testBuildOpts = ""
2540                         if (priority == '1') {
2541                             testBuildOpts = "priority1"
2542                         }
2543
2544                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2545
2546                         // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2547                         buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
2548
2549                         // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
2550                         // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
2551                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2552
2553                         Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2554                     }
2555
2556                     // Archive the build logs from both product and test builds.
2557                     Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
2558
2559                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2560                     // the cleanup task in Jenkins can't remove.
2561                     newJob.with {
2562                         publishers {
2563                             azureVMAgentPostBuildAction {
2564                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2565                             }
2566                         }
2567                     }
2568                     break
2569                 default:
2570                     println("Unknown architecture: ${architecture}");
2571                     assert false
2572                     break
2573             }
2574             break
2575         // editor brace matching: }
2576         default:
2577             println("Unknown os: ${os}");
2578             assert false
2579             break
2580     } // os
2581
2582     return buildCommands
2583 }
2584
2585 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2586 // Returns true if the job should be generated.
2587 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2588 {
2589     // The various "innerloop" jobs are only available as PR triggered.
2590
2591     if (!isPR) {
2592         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
2593             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
2594             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
2595             return true
2596         }
2597
2598         if (isInnerloopTestScenario(scenario)) {
2599             return false
2600         }
2601
2602         if (scenario == 'corefx_innerloop') {
2603             return false
2604         }
2605     }
2606
2607     // Tizen is only supported for armem architecture
2608     if (os == 'Tizen' && architecture != 'armem') {
2609         return false
2610     }
2611
2612     // Filter based on architecture.
2613
2614     switch (architecture) {
2615         case 'arm':
2616             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2617                 return false
2618             }
2619             break
2620         case 'arm64':
2621             if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2622                 return false
2623             }
2624             break
2625         case 'armem':
2626             if (os != 'Tizen') {
2627                 return false
2628             }
2629             break
2630         case 'x86_arm_altjit':
2631         case 'x64_arm64_altjit':
2632             if (os != 'Windows_NT') {
2633                 return false
2634             }
2635             break
2636         case 'x86':
2637             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2638                 return false
2639             }
2640             break
2641         case 'x64':
2642             // Everything implemented
2643             break
2644         default:
2645             println("Unknown architecture: ${architecture}")
2646             assert false
2647             break
2648     }
2649
2650     // Which (Windows) build only jobs are required?
2651
2652     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2653
2654     if (isBuildOnly) {
2655         switch (architecture) {
2656             case 'arm':
2657             case 'arm64':
2658                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2659                 if (!isCoreFxScenario(scenario)) {
2660                     return false
2661                 }
2662                 break
2663             case 'x64':
2664             case 'x86':
2665                 if (!isNormalOrInnerloop) {
2666                     return false
2667                 }
2668                 break
2669             default:
2670                 return false
2671         }
2672     }
2673
2674     // Filter based on scenario.
2675
2676     if (isJitStressScenario(scenario)) {
2677         if (configuration != 'Checked') {
2678             return false
2679         }
2680
2681         def isEnabledOS = (os == 'Windows_NT') ||
2682                           (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2683                           (os == 'Ubuntu' && architecture == 'arm') ||
2684                           (os == 'Ubuntu16.04' && architecture == 'arm64')
2685         if (!isEnabledOS) {
2686             return false
2687         }
2688
2689         switch (architecture) {
2690             case 'x64':
2691             case 'x86_arm_altjit':
2692             case 'x64_arm64_altjit':
2693                 break
2694
2695             case 'x86':
2696                 // x86 ubuntu: no stress modes
2697                 if (os == 'Ubuntu') {
2698                     return false
2699                 }
2700                 break
2701
2702             case 'arm':
2703             case 'arm64':
2704                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2705                 // No "regular" Windows arm corefx jobs, e.g.
2706                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2707                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2708                 if (os == 'Windows_NT') {
2709                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2710                         return false
2711                     }
2712                 }
2713                 else {
2714                     if (!isCoreFxScenario(scenario)) {
2715                         return false
2716                     }
2717                 }
2718                 break
2719
2720             default:
2721                 // armem: no stress jobs for ARM emulator.
2722                 return false
2723         }
2724     }
2725     else if (isR2RScenario(scenario)) {
2726         if (os != 'Windows_NT') {
2727             return false
2728         }
2729
2730         if (isR2RBaselineScenario(scenario)) {
2731             // no need for Debug scenario; Checked is sufficient
2732             if (configuration != 'Checked' && configuration != 'Release') {
2733                 return false
2734             }
2735         }
2736         else if (isR2RStressScenario(scenario)) {
2737             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2738             if (configuration != 'Checked') {
2739                 return false
2740             }
2741         }
2742
2743         switch (architecture) {
2744             case 'arm':
2745             case 'arm64':
2746                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2747                 return false
2748
2749             default:
2750                 break
2751         }
2752     }
2753     else if (isCrossGenComparisonScenario(scenario)) {
2754         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2755     }
2756     else {
2757         // Skip scenarios
2758         switch (scenario) {
2759             case 'ilrt':
2760                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2761                 // the default scenario build
2762                 if (os != 'Windows_NT') {
2763                     return false
2764                 }
2765                 // Only x64 for now
2766                 if (architecture != 'x64') {
2767                     return false
2768                 }
2769                 // Release only
2770                 if (configuration != 'Release') {
2771                     return false
2772                 }
2773                 break
2774             case 'jitdiff':
2775                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2776                     return false
2777                 }
2778                 if (architecture != 'x64') {
2779                     return false
2780                 }
2781                 if (configuration != 'Checked') {
2782                     return false
2783                 }
2784                 break
2785             case 'longgc':
2786             case 'gcsimulator':
2787                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2788                     return false
2789                 }
2790                 if (architecture != 'x64') {
2791                     return false
2792                 }
2793                 if (configuration != 'Release') {
2794                     return false
2795                 }
2796                 break
2797             case 'gc_reliability_framework':
2798             case 'standalone_gc':
2799                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2800                     return false
2801                 }
2802
2803                 if (architecture != 'x64') {
2804                     return false
2805                 }
2806
2807                 if (configuration != 'Release' && configuration != 'Checked') {
2808                     return false
2809                 }
2810                 break
2811             // We only run Windows and Ubuntu x64 Checked for formatting right now
2812             case 'formatting':
2813                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2814                     return false
2815                 }
2816                 if (architecture != 'x64') {
2817                     return false
2818                 }
2819                 if (configuration != 'Checked') {
2820                     return false
2821                 }
2822                 break
2823             case 'illink':
2824                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2825                     return false
2826                 }
2827                 if (architecture != 'x64' && architecture != 'x86') {
2828                     return false
2829                 }
2830                 break
2831             case 'normal':
2832                 // Nothing skipped
2833                 break
2834             case 'innerloop':
2835                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2836                     return false
2837                 }
2838                 break
2839             case 'corefx_innerloop':
2840                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2841                     return false
2842                 }
2843                 if (architecture != 'x64') {
2844                     return false
2845                 }
2846                 break
2847             default:
2848                 println("Unknown scenario: ${scenario}")
2849                 assert false
2850                 break
2851         }
2852     }
2853
2854     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2855     // runtime behavior, not compile-time behavior, are not interesting.
2856     switch (architecture) {
2857         case 'x86_arm_altjit':
2858         case 'x64_arm64_altjit':
2859             if (isGCStressRelatedTesting(scenario)) {
2860                 return false
2861             }
2862             break
2863         default:
2864             break
2865     }
2866
2867     // The job was not filtered out, so we should generate it!
2868     return true
2869 }
2870
2871 Constants.allScenarios.each { scenario ->
2872     [true, false].each { isPR ->
2873         Constants.architectureList.each { architecture ->
2874             Constants.configurationList.each { configuration ->
2875                 Constants.osList.each { os ->
2876                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2877                     // and reset the os to Windows_NT
2878                     def isBuildOnly = false
2879                     if (os == 'Windows_NT_BuildOnly') {
2880                         isBuildOnly = true
2881                         os = 'Windows_NT'
2882                     }
2883
2884                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2885                         return
2886                     }
2887
2888                     // Calculate names
2889                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2890                     def folderName = getJobFolder(scenario)
2891
2892                     // Create the new job
2893                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2894
2895                     addToViews(newJob, false, isPR, architecture, os, configuration, scenario) // isFlowJob == false
2896
2897                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2898
2899                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2900                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2901                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2902
2903                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2904                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2905                     // artifacts produced.
2906                     if ((architecture == 'armem') && (os == 'Tizen')) {
2907                         // Define the Windows Tests and Corefx build job names
2908                         def lowerConfiguration = configuration.toLowerCase()
2909                         def WindowsTestsName = projectFolder + '/' +
2910                                                Utilities.getFullJobName(project,
2911                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2912                                                                         false)
2913                         def fxBranch = getFxBranch(branch)
2914                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2915                                            Utilities.getFolderName(fxBranch)
2916
2917                         def arm_abi = 'armel'
2918                         def corefx_os = 'tizen'
2919
2920                         // Let's use release CoreFX to test checked CoreCLR,
2921                         // because we do not generate checked CoreFX in CoreFX CI yet.
2922                         def corefx_lowerConfiguration = lowerConfiguration
2923                         if (lowerConfiguration == 'checked') {
2924                             corefx_lowerConfiguration = 'release'
2925                         }
2926
2927                         // Copy the Windows test binaries and the Corefx build binaries
2928                         newJob.with {
2929                             steps {
2930                                 copyArtifacts(WindowsTestsName) {
2931                                     includePatterns('bin/tests/tests.zip')
2932                                     buildSelector {
2933                                         latestSuccessful(true)
2934                                     }
2935                                 }
2936                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2937                                     includePatterns('bin/build.tar.gz')
2938                                     buildSelector {
2939                                         latestSuccessful(true)
2940                                     }
2941                                 }
2942                             } // steps
2943                         } // newJob.with
2944                     }
2945
2946                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2947
2948                     newJob.with {
2949                         steps {
2950                             if (os == 'Windows_NT') {
2951                                 buildCommands.each { buildCommand ->
2952                                     batchFile(buildCommand)
2953                                 }
2954                             }
2955                             else {
2956                                 buildCommands.each { buildCommand ->
2957                                     shell(buildCommand)
2958                                 }
2959                             }
2960                         } // steps
2961                     } // newJob.with
2962
2963                 } // os
2964             } // configuration
2965         } // architecture
2966     } // isPR
2967 } // scenario
2968
2969 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2970 // Returns the newly created job.
2971 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2972 {
2973     def osGroup = getOSGroup(os)
2974     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2975
2976     def jobFolder = getJobFolder(scenario)
2977     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2978         parameters {
2979             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2980         }
2981
2982         steps {
2983             // Set up the copies
2984
2985             // Coreclr build we are trying to test
2986             //
2987             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2988
2989             copyArtifacts(inputCoreCLRBuildName) {
2990                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2991                 buildSelector {
2992                     buildNumber('${CORECLR_BUILD}')
2993                 }
2994             }
2995
2996             if (isCoreFxScenario(scenario)) {
2997
2998                 // Only arm/arm64 supported for corefx testing now.
2999                 assert architecture == 'arm' || architecture == 'arm64'
3000
3001                 // Unzip CoreFx runtime
3002                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3003
3004                 // Unzip CoreFx tests.
3005                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3006
3007                 // Add the script to run the corefx tests
3008                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3009                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
3010                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3011                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3012
3013             } else { // !isCoreFxScenario(scenario)
3014
3015                 // Unzip tests.
3016                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3017
3018                 def buildCommands = ""
3019
3020                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3021                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3022                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3023
3024                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3025                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3026
3027                 // For all jobs
3028                 addEnvVariable("CORE_ROOT", coreRootLocation)
3029                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3030                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3031
3032                 // If we are running a stress mode, we'll set those variables as well
3033                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3034                     def stressValues = null
3035                     if (isJitStressScenario(scenario)) {
3036                         stressValues = Constants.jitStressModeScenarios[scenario]
3037                     }
3038                     else {
3039                         stressValues = Constants.r2rStressScenarios[scenario]
3040                     }
3041
3042                     stressValues.each { key, value ->
3043                         addEnvVariable(key, value)
3044                     }
3045                 }
3046
3047                 if (isR2RScenario(scenario)) {
3048                     // Crossgen the framework assemblies.
3049                     buildCommands += """
3050 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3051 @goto skip_PrecompileAssembly
3052
3053 :PrecompileAssembly
3054 @REM Skip mscorlib since it is already precompiled.
3055 @if /I "%3" == "mscorlib.dll" exit /b 0
3056 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3057
3058 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3059 @if "%errorlevel%" == "-2146230517" (
3060     echo %2 is not a managed assembly.
3061 ) else if "%errorlevel%" == "-2146234344" (
3062     echo %2 is not a managed assembly.
3063 ) else if %errorlevel% neq 0 (
3064     echo Unable to precompile %2
3065 ) else (
3066     echo Precompiled %2
3067 )
3068 @exit /b 0
3069
3070 :skip_PrecompileAssembly
3071 """
3072
3073                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3074                     // crossgen on tests before running them.
3075                     addEnvVariable("RunCrossGen", "true")
3076                 } // isR2RScenario(scenario)
3077
3078                 // Create the smarty command
3079                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3080                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3081                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3082                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3083
3084                 // Exclude tests based on scenario.
3085                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3086                     addArchSpecificExclude(architecture, excludeTag)
3087                 }
3088
3089                 if (isInnerloopTestScenario(scenario)) {
3090                     addExclude("pri1")
3091                 }
3092
3093                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3094                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3095                 addExclude("LONG_RUNNING")
3096
3097                 smartyCommand += "/lstFile Tests.lst"
3098
3099                 def testListArch = [
3100                     'arm64': 'arm64',
3101                     'arm': 'arm'
3102                 ]
3103
3104                 def archLocation = testListArch[architecture]
3105
3106                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3107                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3108                 addCommand("${smartyCommand}")
3109
3110                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3111                 // However, we also need to remove all the variables that were set during this batch file, so we
3112                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3113                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3114                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3115                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3116                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3117                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3118
3119                 // ZIP up the smarty output, no matter what the smarty result.
3120                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3121
3122                 addCommand("echo %errorlevel%")
3123                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3124
3125                 // Use the smarty errorlevel as the script errorlevel.
3126                 addCommand("exit /b %__save_smarty_errorlevel%")
3127
3128                 batchFile(buildCommands)
3129             } // non-corefx testing
3130         } // steps
3131     } // job
3132
3133     if (!isCoreFxScenario(scenario)) {
3134         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3135
3136         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3137         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3138         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3139         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3140     }
3141
3142     return newJob
3143 }
3144
3145 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3146 // E.g., non-Windows tests.
3147 // Returns the newly created job.
3148 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3149 {
3150     def lowerConfiguration = configuration.toLowerCase()
3151
3152     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3153     def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3154     def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3155
3156     def doCoreFxTesting = isCoreFxScenario(scenario)
3157
3158     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3159
3160     def osGroup = getOSGroup(os)
3161     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3162
3163     def testOpts = ''
3164     def useServerGC = false
3165
3166     // Enable Server GC for Ubuntu PR builds
3167     // REVIEW: why? Does this apply to all architectures? Why only PR?
3168     if (os == 'Ubuntu' && isPR) {
3169         testOpts += ' --useServerGC'
3170         useServerGC = true
3171     }
3172
3173     if (isR2RScenario(scenario)) {
3174
3175         testOpts += ' --crossgen --runcrossgentests'
3176
3177         if (scenario == 'r2r_jitstress1') {
3178             testOpts += ' --jitstress=1'
3179         }
3180         else if (scenario == 'r2r_jitstress2') {
3181             testOpts += ' --jitstress=2'
3182         }
3183         else if (scenario == 'r2r_jitstress1_tiered') {
3184             testOpts += ' --jitstress=1'
3185         }
3186         else if (scenario == 'r2r_jitstress2_tiered') {
3187             testOpts += ' --jitstress=2'
3188         }
3189         else if (scenario == 'r2r_jitstressregs1') {
3190             testOpts += ' --jitstressregs=1'
3191         }
3192         else if (scenario == 'r2r_jitstressregs2') {
3193             testOpts += ' --jitstressregs=2'
3194         }
3195         else if (scenario == 'r2r_jitstressregs3') {
3196             testOpts += ' --jitstressregs=3'
3197         }
3198         else if (scenario == 'r2r_jitstressregs4') {
3199             testOpts += ' --jitstressregs=4'
3200         }
3201         else if (scenario == 'r2r_jitstressregs8') {
3202             testOpts += ' --jitstressregs=8'
3203         }
3204         else if (scenario == 'r2r_jitstressregs0x10') {
3205             testOpts += ' --jitstressregs=0x10'
3206         }
3207         else if (scenario == 'r2r_jitstressregs0x80') {
3208             testOpts += ' --jitstressregs=0x80'
3209         }
3210         else if (scenario == 'r2r_jitstressregs0x1000') {
3211             testOpts += ' --jitstressregs=0x1000'
3212         }
3213         else if (scenario == 'r2r_jitminopts') {
3214             testOpts += ' --jitminopts'
3215         }
3216         else if (scenario == 'r2r_jitforcerelocs') {
3217             testOpts += ' --jitforcerelocs'
3218         }
3219         else if (scenario == 'r2r_gcstress15') {
3220             testOpts += ' --gcstresslevel=0xF'
3221         }
3222     }
3223     else if (scenario == 'jitdiff') {
3224         testOpts += ' --jitdisasm --crossgen'
3225     }
3226     else if (scenario == 'illink') {
3227         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3228     }
3229     else if (isLongGc(scenario)) {
3230         // Long GC tests behave very poorly when they are not
3231         // the only test running (many of them allocate until OOM).
3232         testOpts += ' --sequential'
3233
3234         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3235         // for running long GC and GCSimulator tests, respectively. We don't use them
3236         // here because using a playlist file produces much more readable output on the CI machines
3237         // and reduces running time.
3238         //
3239         // The Long GC playlist contains all of the tests that are
3240         // going to be run. The GCSimulator playlist contains all of
3241         // the GC simulator tests.
3242         if (scenario == 'longgc') {
3243             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3244         }
3245         else if (scenario == 'gcsimulator') {
3246             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3247         }
3248     }
3249     else if (isGcReliabilityFramework(scenario)) {
3250         testOpts += ' --build-overlay-only'
3251     }
3252     else if (scenario == 'standalone_gc') {
3253         if (osGroup == 'OSX') {
3254             testOpts += ' --gcname=libclrgc.dylib'
3255         }
3256         else if (osGroup == 'Linux') {
3257             testOpts += ' --gcname=libclrgc.so'
3258         }
3259         else {
3260             println("Unexpected OS group: ${osGroup} for os ${os}")
3261             assert false
3262         }
3263     }
3264
3265     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3266     // will be null in this case.
3267
3268     def jobFolder = getJobFolder(scenario)
3269     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3270         parameters {
3271             if (inputTestsBuildName != null) {
3272                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3273             }
3274             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3275         }
3276
3277         steps {
3278             // Set up the copies
3279
3280             // Coreclr build containing the tests and mscorlib
3281             // pri1 jobs still need to copy windows_nt built tests
3282             if (inputTestsBuildName != null) {
3283                 copyArtifacts(inputTestsBuildName) {
3284                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3285                     buildSelector {
3286                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3287                     }
3288                 }
3289             }
3290
3291             // Coreclr build we are trying to test
3292             //
3293             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3294             //
3295             // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3296             // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3297
3298             if (!isUbuntuArm64Job) {
3299                 copyArtifacts(inputCoreCLRBuildName) {
3300                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3301                     buildSelector {
3302                         buildNumber('${CORECLR_BUILD}')
3303                     }
3304                 }
3305             }
3306
3307             if (isUbuntuArmJob) {
3308                 // Add some useful information to the log file. Ignore return codes.
3309                 shell("uname -a || true")
3310             }
3311
3312             if (isUbuntuArm64Job) {
3313                 // Copy the required artifacts directly, using wget, e.g.:
3314                 //
3315                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3316                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3317                 //
3318                 // parameterized as:
3319                 //
3320                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3321                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3322                 //
3323                 // CoreFX example artifact URLs:
3324                 //
3325                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3326                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3327                 //
3328                 // Note that the source might be in a "jitstress" folder.
3329                 //
3330                 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3331                 //
3332                 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3333
3334                 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3335
3336                 def mungedProjectName = Utilities.getFolderName(project)
3337                 def mungedBranchName = Utilities.getFolderName(branch)
3338
3339                 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3340                 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3341                 if (doCoreFxTesting || doCrossGenComparison) {
3342                     // These depend on unique builds for each scenario
3343                     inputCoreCLRBuildScenario = scenario
3344                 }
3345                 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3346                 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3347
3348                 // Need to add the sub-folder if necessary.
3349                 def inputJobPath = "job/${inputJobName}"
3350                 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3351                 if (folderName != '') {
3352                     inputJobPath = "job/${folderName}/job/${inputJobName}"
3353                 }
3354
3355                 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3356
3357                 if (doCoreFxTesting) {
3358                     shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3359                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3360                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3361                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/run-test.sh")
3362                     shell("chmod +x ${workspaceRelativeFxRootLinux}/run-test.sh")
3363                 }
3364                 else {
3365                     shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
3366                     shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
3367                 }
3368             }
3369
3370             if (architecture == 'x86') {
3371                 shell("mkdir ./bin/CoreFxNative")
3372
3373                 def fxBranch = getFxBranch(branch)
3374                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3375
3376                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3377                     includePatterns('bin/build.tar.gz')
3378                     targetDirectory('bin/CoreFxNative')
3379                     buildSelector {
3380                         latestSuccessful(true)
3381                     }
3382                 }
3383
3384                 shell("mkdir ./bin/CoreFxBinDir")
3385                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3386             }
3387
3388             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3389             if (doCoreFxTesting) {
3390                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3391                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3392             }
3393             else if (architecture != 'arm64') {
3394                 // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
3395                 //
3396                 // Unzip the tests first.  Exit with 0
3397                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3398                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3399             }
3400
3401             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3402             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3403             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3404             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3405             // copied correctly.
3406             if (!doCoreFxTesting) {
3407                 if (isUbuntuArmJob) {
3408                     if (architecture == 'arm') {
3409                         shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3410                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3411                     }
3412                     else {
3413                         assert architecture == 'arm64'
3414                         shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
3415
3416                         // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
3417                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3418                     }
3419                 }
3420                 else {
3421                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3422                 }
3423             }
3424
3425             // Execute the tests
3426             def runDocker = isNeedDocker(architecture, os, false)
3427             def dockerPrefix = ""
3428             def dockerCmd = ""
3429             if (runDocker) {
3430                 def dockerImage = getDockerImageName(architecture, os, false)
3431                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3432                 dockerCmd = dockerPrefix + "${dockerImage} "
3433             }
3434
3435             // If we are running a stress mode, we'll set those variables first.
3436             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3437             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3438                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3439                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3440                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3441                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3442                 shell("${envScriptCmds}")
3443                 testOpts += " --test-env=${scriptFileName}"
3444             }
3445
3446             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3447             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3448             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3449             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3450             if (isGCStressRelatedTesting(scenario)) {
3451                 if (architecture == 'x64') {
3452                     shell('./init-tools.sh')
3453                 }
3454             }
3455
3456             if (doCoreFxTesting) {
3457                 shell("""\
3458 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3459             }
3460             else {
3461                 def runScript = "${dockerCmd}./tests/runtest.sh"
3462
3463                 // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
3464
3465                 shell("""\
3466 ${runScript} \\
3467     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3468     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3469     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3470     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3471             }
3472
3473             if (isGcReliabilityFramework(scenario)) {
3474                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3475                 if (useServerGC) {
3476                     if (runDocker) {
3477                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3478                     }
3479                     else {
3480                         shell("export COMPlus_gcServer=1")
3481                     }
3482                 }
3483
3484                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3485             }
3486         } // steps
3487     } // job
3488
3489     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3490     if (os in ['Ubuntu']) {
3491         SummaryBuilder summaries = new SummaryBuilder()
3492         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3493         summaries.emit(newJob)
3494     }
3495
3496     if (doCoreFxTesting) {
3497         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3498         if ((os == "Ubuntu") && (architecture == 'arm')) {
3499             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3500             //
3501             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3502             //
3503             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3504             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3505         }
3506         else {
3507             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3508         }
3509     }
3510     else {
3511         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3512     }
3513
3514     return newJob
3515 }
3516
3517 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3518 {
3519     assert isCrossGenComparisonScenario(scenario)
3520
3521     def osGroup = getOSGroup(os)
3522     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3523
3524     def workspaceRelativeResultsDir = "_"
3525     def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3526
3527     def jobFolder = getJobFolder(scenario)
3528     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3529         parameters {
3530             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3531         }
3532
3533         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3534
3535         steps {
3536             copyArtifacts(inputCoreCLRBuildName) {
3537                 includePatterns("${workspaceRelativeArtifactsArchive}")
3538                 buildSelector {
3539                     buildNumber('${CORECLR_BUILD}')
3540                 }
3541             }
3542
3543             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3544
3545             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3546             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3547             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3548             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3549
3550             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3551             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3552
3553             shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3554             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3555             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3556
3557             getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3558                 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3559                 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3560             } // crossArch
3561         } // steps
3562     }  // job
3563
3564     Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**")
3565     getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3566         def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3567         Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**")
3568     } // crossArch
3569
3570     return newJob
3571 }
3572
3573 // Create a test job that will be used by a flow job.
3574 // Returns the newly created job.
3575 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3576 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3577 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3578 {
3579     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3580
3581     def newJob = null
3582     if (windowsArmJob) {
3583         assert inputTestsBuildName == null
3584         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3585     }
3586     else if (isCrossGenComparisonScenario(scenario)) {
3587         assert inputTestsBuildName == null
3588         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3589     }
3590     else {
3591         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3592     }
3593
3594     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3595
3596     if (scenario == 'jitdiff') {
3597         def osGroup = getOSGroup(os)
3598         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3599     }
3600
3601     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3602     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3603
3604     return newJob
3605 }
3606
3607 // Create a flow job to tie together a build job with the given test job.
3608 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3609 // Returns the new flow job.
3610 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3611 {
3612     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3613     // Linux CoreCLR test
3614     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3615     def jobFolder = getJobFolder(scenario)
3616
3617     def newFlowJob = null
3618
3619     if (inputTestsBuildName == null) {
3620         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3621                         buildFlow("""\
3622 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3623
3624 // And then build the test build
3625 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3626 """)
3627         }
3628         JobReport.Report.addReference(inputCoreCLRBuildName)
3629         JobReport.Report.addReference(fullTestJobName)
3630     }
3631     else {
3632         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3633                         buildFlow("""\
3634 // Build the input jobs in parallel
3635 parallel (
3636 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3637 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3638 )
3639
3640 // And then build the test build
3641 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3642                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3643 """)
3644         }
3645         JobReport.Report.addReference(inputCoreCLRBuildName)
3646         JobReport.Report.addReference(inputTestsBuildName)
3647         JobReport.Report.addReference(fullTestJobName)
3648     }
3649
3650     addToViews(newFlowJob, true, isPR, architecture, os, configuration, scenario) // isFlowJob = true
3651
3652     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3653
3654     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3655     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3656
3657     return newFlowJob
3658 }
3659
3660 // Determine if we should generate a flow job for the given parameters.
3661 // Returns true if the job should be generated.
3662 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3663 {
3664     // The various "innerloop" jobs are only available as PR triggered.
3665
3666     if (!isPR) {
3667         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
3668             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
3669             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
3670             return true
3671         }
3672
3673         if (isInnerloopTestScenario(scenario)) {
3674             return false
3675         }
3676
3677         if (scenario == 'corefx_innerloop') {
3678             return false
3679         }
3680     }
3681
3682     // Filter based on OS and architecture.
3683
3684     switch (architecture) {
3685         case 'arm':
3686             if (os != "Ubuntu" && os != "Windows_NT") {
3687                 return false
3688             }
3689             break
3690         case 'arm64':
3691             if (os != "Ubuntu16.04" && os != "Windows_NT") {
3692                 return false
3693             }
3694             break
3695         case 'x86':
3696             if (os != "Ubuntu") {
3697                 return false
3698             }
3699             break
3700         case 'x64':
3701             if (!(os in Constants.crossList)) {
3702                 return false
3703             }
3704             if (os == "Windows_NT") {
3705                 return false
3706             }
3707             break
3708         case 'armem':
3709         case 'x86_arm_altjit':
3710         case 'x64_arm64_altjit':
3711             // No flow jobs
3712             return false
3713         default:
3714             println("Unknown architecture: ${architecture}")
3715             assert false
3716             break
3717     }
3718
3719     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3720
3721     // Filter based on scenario in OS.
3722
3723     if (os == 'Windows_NT') {
3724         assert architecture == 'arm' || architecture == 'arm64'
3725         if (!isArmWindowsScenario(scenario)) {
3726             return false
3727         }
3728         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3729             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3730             // To do that, it doesn't require a flow job.
3731             return false
3732         }
3733     }
3734     else {
3735         // Non-Windows
3736         if (architecture == 'arm') {
3737             if (!(scenario in Constants.validLinuxArmScenarios)) {
3738                 return false
3739             }
3740         }
3741         else if (architecture == 'arm64') {
3742             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3743                 return false
3744             }
3745         }
3746         else if (architecture == 'x86') {
3747             // Linux/x86 only want innerloop and default test
3748             if (!isNormalOrInnerloop) {
3749                 return false
3750             }
3751         }
3752         else if (architecture == 'x64') {
3753             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3754             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3755             // build from a Windows machine.
3756             if (isCoreFxScenario(scenario)) {
3757                 return false
3758             }
3759         }
3760     }
3761
3762     // For CentOS, we only want Checked/Release builds.
3763     if (os == 'CentOS7.1') {
3764         if (configuration != 'Checked' && configuration != 'Release') {
3765             return false
3766         }
3767         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3768             return false
3769         }
3770     }
3771
3772     // For RedHat and Debian, we only do Release builds.
3773     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3774         if (configuration != 'Release') {
3775             return false
3776         }
3777         if (!isNormalOrInnerloop) {
3778             return false
3779         }
3780     }
3781
3782     // Next, filter based on scenario.
3783
3784     if (isJitStressScenario(scenario)) {
3785         if (configuration != 'Checked') {
3786             return false
3787         }
3788     }
3789     else if (isR2RBaselineScenario(scenario)) {
3790         if (configuration != 'Checked' && configuration != 'Release') {
3791             return false
3792         }
3793     }
3794     else if (isR2RStressScenario(scenario)) {
3795         if (configuration != 'Checked') {
3796             return false
3797         }
3798     }
3799     else if (isCrossGenComparisonScenario(scenario)) {
3800         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3801     }
3802     else {
3803         // Skip scenarios
3804         switch (scenario) {
3805             case 'ilrt':
3806             case 'longgc':
3807             case 'gcsimulator':
3808                 // Long GC tests take a long time on non-Release builds
3809                 // ilrt is also Release only
3810                 if (configuration != 'Release') {
3811                     return false
3812                 }
3813                 break
3814
3815             case 'jitdiff':
3816                 if (configuration != 'Checked') {
3817                     return false
3818                 }
3819                 break
3820
3821             case 'gc_reliability_framework':
3822             case 'standalone_gc':
3823                 if (configuration != 'Release' && configuration != 'Checked') {
3824                     return false
3825                 }
3826                 break
3827
3828             case 'formatting':
3829                 return false
3830
3831             case 'illink':
3832                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3833                     return false
3834                 }
3835                 break
3836
3837             case 'normal':
3838                 // Nothing skipped
3839                 break
3840
3841             case 'innerloop':
3842                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3843                     return false
3844                 }
3845                 break
3846
3847             case 'corefx_innerloop':
3848                 // No flow job needed
3849                 return false
3850
3851             default:
3852                 println("Unknown scenario: ${scenario}")
3853                 assert false
3854                 break
3855         }
3856     }
3857
3858     // The job was not filtered out, so we should generate it!
3859     return true
3860 }
3861
3862 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3863 Constants.allScenarios.each { scenario ->
3864     [true, false].each { isPR ->
3865         Constants.architectureList.each { architecture ->
3866             Constants.configurationList.each { configuration ->
3867                 Constants.osList.each { os ->
3868
3869                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3870                         return
3871                     }
3872
3873                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3874                     def doCoreFxTesting = isCoreFxScenario(scenario)
3875                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3876
3877                     // Figure out the job name of the CoreCLR build the test will depend on.
3878
3879                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3880                     def inputCoreCLRBuildIsBuildOnly = false
3881                     if (doCoreFxTesting) {
3882                         // Every CoreFx test depends on its own unique build.
3883                         inputCoreCLRBuildScenario = scenario
3884                         if (windowsArmJob) {
3885                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3886                             inputCoreCLRBuildIsBuildOnly = true
3887                         }
3888                     }
3889                     if (doCrossGenComparison) {
3890                         inputCoreCLRBuildScenario = scenario
3891                     }
3892
3893                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3894                     def inputCoreCLRBuildName = projectFolder + '/' +
3895                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3896
3897                     // Figure out the name of the build job that the test job will depend on.
3898                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3899                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3900                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3901                     // CoreFX build creates the tests.
3902
3903                     def inputTestsBuildName = null
3904
3905                     // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
3906                     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3907
3908                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job) {
3909                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3910
3911                         def inputTestsBuildArch = architecture
3912                         if (architecture == "arm") {
3913                             // Use the x86 test build for arm unix
3914                             inputTestsBuildArch = "x86"
3915                         }
3916
3917                         def inputTestsBuildIsBuildOnly = true
3918
3919                         inputTestsBuildName = projectFolder + '/' +
3920                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3921                     }
3922
3923                     // =============================================================================================
3924                     // Create the test job
3925                     // =============================================================================================
3926
3927                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3928
3929                     // =============================================================================================
3930                     // Create a build flow to join together the build and tests required to run this test.
3931                     // =============================================================================================
3932
3933                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3934                         // Do not create the flow job for RHEL jobs.
3935                         return
3936                     }
3937
3938                     def fullTestJobName = projectFolder + '/' + testJob.name
3939                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3940
3941                 } // os
3942             } // configuration
3943         } // architecture
3944     } // isPR
3945 } // scenario
3946
3947 JobReport.Report.generateJobReport(out)
3948
3949 // Make the call to generate the help job
3950 Utilities.createHelperJob(this, project, branch,
3951     "Welcome to the ${project} Repository",  // This is prepended to the help message
3952     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3953
3954 Utilities.addCROSSCheck(this, project, branch)