netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu'      : 'Linux',
  21                       'Ubuntu16.04' : 'Linux',
  22                       'Ubuntu16.10' : 'Linux',
  23                       'RHEL7.2'     : 'Linux',
  24                       'Debian8.4'   : 'Linux',
  25                       'Fedora24'    : 'Linux',
  26                       'CentOS7.1'   : 'Linux',
  27                       'Tizen'       : 'Linux',
  28                       'OSX10.12'    : 'OSX',
  29                       'Windows_NT'  : 'Windows_NT']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitectures(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return ['x86','x64']
  39
  40         case 'arm64':
  41             return ['x64']
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited Windows ARM64 hardware (used for ARM/ARM64 testing) and Linux/arm32 and Linux/arm64 hardware.
  51     // So only allow certain branches to use it.
  52     def static LimitedHardwareBranches = [
  53                'master']
  54
  55     // Innerloop build OS's
  56     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  57     // test execution in the build flow runs.  It generates the exact same build
  58     // as Windows_NT but without running the tests.
  59     def static osList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'Windows_NT_BuildOnly',
  65                'CentOS7.1',
  66                'RHEL7.2',
  67                'Ubuntu16.04',
  68                'Ubuntu16.10',
  69                'Fedora24',
  70                'Tizen']
  71
  72     def static crossList = [
  73                'Ubuntu',
  74                'Debian8.4',
  75                'OSX10.12',
  76                'Windows_NT',
  77                'CentOS7.1',
  78                'RHEL7.2']
  79
  80     // This is a set of JIT stress modes combined with the set of variables that
  81     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  82     // the values are the environment variables
  83     def static jitStressModeScenarios = [
  84                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  85                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  86                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  87                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  88                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  89                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  90                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  91                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  92                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  93                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  94                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  95                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  96                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  97                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  98                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  99                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 100                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 101                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 102                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 103                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 104                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 105                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 106                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 107                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 108                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 109                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 110                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 111                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 112                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 113                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 114                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 115                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 116                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 117                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 118                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 119                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 120                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 121                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 122                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 123                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 124                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 125                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 126                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 127                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 128                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 129                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 130                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 131                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 132                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 133                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 134                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 135                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 136                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 137                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 138                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 139                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 140     ]
 141
 142     // This is a set of ReadyToRun stress scenarios
 143     def static r2rStressScenarios = [
 144                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 145                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 146                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 147                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 148                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 149                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 150                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 151                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 152                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 153                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 154                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 155                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 156                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 157                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 158                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 159                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 160     ]
 161
 162     // This is the basic set of scenarios
 163     def static basicScenarios = [
 164                'innerloop',
 165                'normal',
 166                'ilrt',
 167                'r2r',
 168                'longgc',
 169                'formatting',
 170                'gcsimulator',
 171                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 172                'standalone_gc',
 173                'gc_reliability_framework',
 174                'illink',
 175                'corefx_innerloop',
 176                'crossgen_comparison']
 177
 178     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 179
 180     // Valid PR trigger combinations.
 181     def static prTriggeredValidInnerLoopCombos = [
 182         'Windows_NT': [
 183             'x64': [
 184                 'Checked'
 185             ],
 186             'x86': [
 187                 'Checked',
 188                 'Release'
 189             ],
 190             'arm': [
 191                 'Debug',
 192                 'Checked'
 193             ],
 194             'arm64': [
 195                 'Debug',
 196                 'Checked'
 197             ]
 198         ],
 199         'Windows_NT_BuildOnly': [
 200             'x64': [
 201                 'Checked',
 202                 'Release'
 203             ],
 204             'x86': [
 205                 'Checked',
 206                 'Release'
 207             ],
 208             'arm': [
 209                 'Checked'
 210             ],
 211         ],
 212         'Ubuntu': [
 213             'x64': [
 214                 'Checked'
 215             ],
 216             'arm': [
 217                 'Checked'
 218             ]
 219         ],
 220         'Ubuntu16.04': [
 221             'arm64': [
 222                 'Checked'
 223             ]
 224         ],
 225         'CentOS7.1': [
 226             'x64': [
 227                 'Debug',
 228                 'Checked'
 229             ]
 230         ],
 231         'OSX10.12': [
 232             'x64': [
 233                 'Checked'
 234             ]
 235         ],
 236         'Tizen': [
 237             'armem': [
 238                 'Checked'
 239             ]
 240         ]
 241     ]
 242
 243     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 244     // to Tests.lst file categories to exclude.
 245     //
 246     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 247     // and with the same values, with some commented out, for easier maintenance.
 248     //
 249     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 250     //
 251     def static validArmWindowsScenarios = [
 252                'innerloop':                              [],
 253                'normal':                                 [],
 254                // 'ilrt'
 255                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 256                // 'longgc'
 257                // 'formatting'
 258                // 'gcsimulator'
 259                // 'jitdiff'
 260                // 'standalone_gc'
 261                // 'gc_reliability_framework'
 262                // 'illink'
 263                // 'corefx_innerloop'
 264                // 'crossgen_comparison'
 265                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 277                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 278                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 279                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 280                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 281                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 282                'tieredcompilation':                      [],
 283                'no_tiered_compilation':                  [],
 284                'no_tiered_compilation_innerloop':        [],
 285                'forcerelocs':                            [],
 286                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 290                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 293                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 294                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 306                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 307                // 'jitsse2only'                          // Only relevant to xarch
 308                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 309                // 'jitincompletehwintrinsic'
 310                // 'jitx86hwintrinsicnoavx'
 311                // 'jitx86hwintrinsicnoavx2'
 312                // 'jitx86hwintrinsicnosimd'
 313                // 'jitnox86hwintrinsic'
 314                'corefx_baseline':                        [],    // corefx tests don't use smarty
 315                'corefx_minopts':                         [],    // corefx tests don't use smarty
 316                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 317                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 318                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 319                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 320                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 321                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 322                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 323                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 324                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 325                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 326                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 327                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 328                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 329                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 330                'heapverify1':                            [],
 331                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 332                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 333                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 334                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 336                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 337
 338                //
 339                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 340                //
 341
 342                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 343                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 344                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 345                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 346                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 347                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 348                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 349                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 350                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 351                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 352                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 353                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 354                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 355                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 356                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 357                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 358                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 359                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 360                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 361     ]
 362
 363     def static validLinuxArmScenarios = [
 364                'innerloop',
 365                'normal',
 366                // 'ilrt'
 367                'r2r',
 368                // 'longgc'
 369                // 'formatting'
 370                // 'gcsimulator'
 371                // 'jitdiff'
 372                // 'standalone_gc'
 373                // 'gc_reliability_framework'
 374                // 'illink'
 375                // 'corefx_innerloop'
 376                'crossgen_comparison',
 377                'r2r_jitstress1',
 378                'r2r_jitstress2',
 379                'r2r_jitstress1_tiered',
 380                'r2r_jitstress2_tiered',
 381                'r2r_jitstressregs1',
 382                'r2r_jitstressregs2',
 383                'r2r_jitstressregs3',
 384                'r2r_jitstressregs4',
 385                'r2r_jitstressregs8',
 386                'r2r_jitstressregs0x10',
 387                'r2r_jitstressregs0x80',
 388                'r2r_jitstressregs0x1000',
 389                'r2r_jitminopts',
 390                'r2r_jitforcerelocs',
 391                'r2r_gcstress15',
 392                'r2r_no_tiered_compilation',
 393                'minopts',
 394                'tieredcompilation',
 395                'no_tiered_compilation',
 396                'no_tiered_compilation_innerloop',
 397                'forcerelocs',
 398                'jitstress1',
 399                'jitstress2',
 400                'jitstress1_tiered',
 401                'jitstress2_tiered',
 402                'jitstressregs1',
 403                'jitstressregs2',
 404                'jitstressregs3',
 405                'jitstressregs4',
 406                'jitstressregs8',
 407                'jitstressregs0x10',
 408                'jitstressregs0x80',
 409                'jitstressregs0x1000',
 410                'jitstress2_jitstressregs1',
 411                'jitstress2_jitstressregs2',
 412                'jitstress2_jitstressregs3',
 413                'jitstress2_jitstressregs4',
 414                'jitstress2_jitstressregs8',
 415                'jitstress2_jitstressregs0x10',
 416                'jitstress2_jitstressregs0x80',
 417                'jitstress2_jitstressregs0x1000',
 418                'tailcallstress',
 419                // 'jitsse2only'                          // Only relevant to xarch
 420                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 421                // 'jitincompletehwintrinsic'
 422                // 'jitx86hwintrinsicnoavx'
 423                // 'jitx86hwintrinsicnoavx2'
 424                // 'jitx86hwintrinsicnosimd'
 425                // 'jitnox86hwintrinsic'
 426                'corefx_baseline',
 427                'corefx_minopts',
 428                'corefx_tieredcompilation',
 429                'corefx_jitstress1',
 430                'corefx_jitstress2',
 431                'corefx_jitstressregs1',
 432                'corefx_jitstressregs2',
 433                'corefx_jitstressregs3',
 434                'corefx_jitstressregs4',
 435                'corefx_jitstressregs8',
 436                'corefx_jitstressregs0x10',
 437                'corefx_jitstressregs0x80',
 438                'corefx_jitstressregs0x1000',
 439                'gcstress0x3',
 440                'gcstress0xc',
 441                'zapdisable',
 442                'heapverify1',
 443                'gcstress0xc_zapdisable',
 444                'gcstress0xc_zapdisable_jitstress2',
 445                'gcstress0xc_zapdisable_heapverify1',
 446                'gcstress0xc_jitstress1',
 447                'gcstress0xc_jitstress2',
 448                'gcstress0xc_minopts_heapverify1'
 449     ]
 450
 451     def static validLinuxArm64Scenarios = [
 452                'innerloop',
 453                'normal',
 454                // 'ilrt'
 455                'r2r',
 456                // 'longgc'
 457                // 'formatting'
 458                // 'gcsimulator'
 459                // 'jitdiff'
 460                // 'standalone_gc'
 461                // 'gc_reliability_framework'
 462                // 'illink'
 463                // 'corefx_innerloop'
 464                // 'crossgen_comparison'
 465                'r2r_jitstress1',
 466                'r2r_jitstress2',
 467                'r2r_jitstress1_tiered',
 468                'r2r_jitstress2_tiered',
 469                'r2r_jitstressregs1',
 470                'r2r_jitstressregs2',
 471                'r2r_jitstressregs3',
 472                'r2r_jitstressregs4',
 473                'r2r_jitstressregs8',
 474                'r2r_jitstressregs0x10',
 475                'r2r_jitstressregs0x80',
 476                'r2r_jitstressregs0x1000',
 477                'r2r_jitminopts',
 478                'r2r_jitforcerelocs',
 479                'r2r_gcstress15',
 480                'r2r_no_tiered_compilation',
 481                'minopts',
 482                'tieredcompilation',
 483                'no_tiered_compilation',
 484                'no_tiered_compilation_innerloop',
 485                'forcerelocs',
 486                'jitstress1',
 487                'jitstress2',
 488                'jitstress1_tiered',
 489                'jitstress2_tiered',
 490                'jitstressregs1',
 491                'jitstressregs2',
 492                'jitstressregs3',
 493                'jitstressregs4',
 494                'jitstressregs8',
 495                'jitstressregs0x10',
 496                'jitstressregs0x80',
 497                'jitstressregs0x1000',
 498                'jitstress2_jitstressregs1',
 499                'jitstress2_jitstressregs2',
 500                'jitstress2_jitstressregs3',
 501                'jitstress2_jitstressregs4',
 502                'jitstress2_jitstressregs8',
 503                'jitstress2_jitstressregs0x10',
 504                'jitstress2_jitstressregs0x80',
 505                'jitstress2_jitstressregs0x1000',
 506                'tailcallstress',
 507                // 'jitsse2only'                         // Only relevant to xarch
 508                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 509                // 'jitincompletehwintrinsic'
 510                // 'jitx86hwintrinsicnoavx'
 511                // 'jitx86hwintrinsicnoavx2'
 512                // 'jitx86hwintrinsicnosimd'
 513                // 'jitnox86hwintrinsic'
 514                'corefx_baseline',
 515                'corefx_minopts',
 516                'corefx_tieredcompilation',
 517                'corefx_jitstress1',
 518                'corefx_jitstress2',
 519                'corefx_jitstressregs1',
 520                'corefx_jitstressregs2',
 521                'corefx_jitstressregs3',
 522                'corefx_jitstressregs4',
 523                'corefx_jitstressregs8',
 524                'corefx_jitstressregs0x10',
 525                'corefx_jitstressregs0x80',
 526                'corefx_jitstressregs0x1000',
 527                'gcstress0x3',
 528                'gcstress0xc',
 529                'zapdisable',
 530                'heapverify1',
 531                'gcstress0xc_zapdisable',
 532                'gcstress0xc_zapdisable_jitstress2',
 533                'gcstress0xc_zapdisable_heapverify1',
 534                'gcstress0xc_jitstress1',
 535                'gcstress0xc_jitstress2',
 536                'gcstress0xc_minopts_heapverify1'
 537     ]
 538
 539     def static configurationList = ['Debug', 'Checked', 'Release']
 540
 541     // This is the set of architectures
 542     // Some of these are pseudo-architectures:
 543     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 544     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 545     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 546     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 547
 548     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 549     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 550 }
 551
 552 // **************************************************************
 553 // Create some specific views
 554 //
 555 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 556 // views based on a single regular expression. These views will be generated by adding a
 557 // specific set of jobs to them.
 558 //
 559 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 560 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 561 // those are required, don't add them (which simplifies the view pages, as well).
 562 // **************************************************************
 563
 564 class Views {
 565     def static MergeJobView = null
 566     def static PeriodicJobView = null
 567     def static ArchitectureViews = [:]
 568     def static OSViews = [:]
 569 }
 570
 571 // MergeJobView: include all jobs that execute when a PR change is merged.
 572 Views.MergeJobView = listView('Merge') {
 573     recurse()
 574     columns {
 575         status()
 576         weather()
 577         name()
 578         lastSuccess()
 579         lastFailure()
 580         lastDuration()
 581         buildButton()
 582     }
 583 }
 584
 585 // PeriodicJobView: include all jobs that execute on a schedule
 586 Views.PeriodicJobView = listView('Periodic') {
 587     recurse()
 588     columns {
 589         status()
 590         weather()
 591         name()
 592         lastSuccess()
 593         lastFailure()
 594         lastDuration()
 595         buildButton()
 596     }
 597 }
 598
 599 // Create a view for non-PR jobs for each architecture.
 600 Constants.architectureList.each { architecture ->
 601     Views.ArchitectureViews[architecture] = listView(architecture) {
 602         recurse()
 603         columns {
 604             status()
 605             weather()
 606             name()
 607             lastSuccess()
 608             lastFailure()
 609             lastDuration()
 610             buildButton()
 611         }
 612     }
 613 }
 614
 615 // Create a view for non-PR jobs for each OS.
 616 Constants.osList.each { os ->
 617     // Don't create one for the special 'Windows_NT_BuildOnly'
 618     if (os == 'Windows_NT_BuildOnly') {
 619         return
 620     }
 621     Views.OSViews[os] = listView(os) {
 622         recurse()
 623         columns {
 624             status()
 625             weather()
 626             name()
 627             lastSuccess()
 628             lastFailure()
 629             lastDuration()
 630             buildButton()
 631         }
 632     }
 633 }
 634
 635 def static addToMergeView(def job) {
 636     Views.MergeJobView.with {
 637         jobs {
 638             name(job.name)
 639         }
 640     }
 641 }
 642
 643 def static addToPeriodicView(def job) {
 644     Views.PeriodicJobView.with {
 645         jobs {
 646             name(job.name)
 647         }
 648     }
 649 }
 650
 651 def static addToViews(def job, def isFlowJob, def isPR, def architecture, def os, def configuration, def scenario) {
 652     if (isPR) {
 653         // No views want PR jobs currently.
 654         return
 655     }
 656
 657     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
 658     // "top-level" jobs. Build only jobs are such jobs.
 659     if (os == 'Windows_NT_BuildOnly') {
 660         return
 661     }
 662
 663     if (!isFlowJob) {
 664         // For non-flow jobs, which ones are only used by flow jobs?
 665         if ((architecture == 'arm') || (architecture == 'arm64')) {
 666             if (isCoreFxScenario(scenario)) {
 667                 // We have corefx-specific scenario builds for each of the runs, but these are driven by flow jobs.
 668                 return
 669             }
 670
 671             // We're left with the basic normal/innerloop builds. We might want to remove these from the views also, if desired.
 672             // However, there are a few, like the Debug Build, that is build only, not "Build and Test", that we should leave.
 673         }
 674     }
 675
 676     // Add to architecture view.
 677     Views.ArchitectureViews[architecture].with {
 678         jobs {
 679             name(job.name)
 680         }
 681     }
 682
 683     // Add to OS view.
 684     Views.OSViews[os].with {
 685         jobs {
 686             name(job.name)
 687         }
 688     }
 689 }
 690
 691 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 692     addToPeriodicView(job)
 693     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 694 }
 695
 696 def static addGithubPushTriggerHelper(def job) {
 697     addToMergeView(job)
 698     Utilities.addGithubPushTrigger(job)
 699 }
 700
 701
 702 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 703     assert os instanceof String
 704     assert architecture instanceof String
 705
 706     def armArches = ['arm', 'armem', 'arm64']
 707
 708     if (!(architecture in armArches)) {
 709         assert options == null
 710         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 711
 712         return
 713     }
 714
 715     // This is an arm(64) job.
 716     //
 717     // There are several options.
 718     //
 719     // Windows_NT
 720     //
 721     // Arm32 (Build) -> latest-arm64
 722     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 723     // Arm32 (Test)  -> arm64-windows_nt
 724     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 725     //
 726     // Arm64 (Build) -> latest-arm64
 727     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 728     // Arm64 (Test)  -> arm64-windows_nt
 729     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 730     //
 731     // Ubuntu
 732     //
 733     // Arm32 emulator (Build, Test) -> arm-cross-latest
 734     //       |-> os == "Tizen" && (architecture == "armem")
 735     //
 736     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 737     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 738     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 739     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 740     // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
 741     //       |-> os == "Ubuntu" && (architecture == "arm")
 742     //
 743     // Arm64 (Build) -> arm64-cross-latest
 744     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
 745     // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
 746     //       |-> os != "Windows_NT" && architecture == "arm64"
 747     //
 748     // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
 749     // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
 750     //
 751     // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
 752     // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 753
 754     // This has to be a arm arch
 755     assert architecture in armArches
 756     if (os == "Windows_NT") {
 757         // arm32/arm64 Windows jobs share the same machines for now
 758         def isBuild = options['use_arm64_build_machine'] == true
 759
 760         if (isBuild == true) {
 761             job.with {
 762                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 763             }
 764         } else {
 765             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 766         }
 767     } else {
 768         assert os != 'Windows_NT'
 769
 770         if (architecture == 'armem') {
 771             // arm emulator (Tizen). Build and test on same machine,
 772             // using Docker.
 773             assert os == 'Tizen'
 774             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 775         }
 776         else {
 777             // arm/arm64 Ubuntu on hardware.
 778             assert architecture == 'arm' || architecture == 'arm64'
 779             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 780             def isBuild = (options != null) && (options['is_build_job'] == true)
 781             if (isFlow || isBuild) {
 782                 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 783                 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
 784                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 785             } else {
 786                 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
 787                 if (architecture == 'arm64') {
 788                     assert os == 'Ubuntu16.04'
 789                     job.with {
 790                         label('Ubuntu.1604.Arm64.Open')
 791                     }
 792                 }
 793                 else {
 794                     assert os == 'Ubuntu'
 795                     job.with {
 796                         label('ubuntu.1404.arm32.open')
 797                     }
 798                 }
 799             }
 800         }
 801     }
 802 }
 803
 804 // setJobMachineAffinity: compute the machine affinity options for a job,
 805 // then set the job with those affinity options.
 806 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 807 {
 808     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 809            (!isBuildJob && isTestJob  && !isFlowJob) ||
 810            (!isBuildJob && !isTestJob && isFlowJob)
 811
 812     def affinityOptions = null
 813     def affinityArchitecture = architecture
 814
 815     if (os == "Windows_NT") {
 816         if (architecture in Constants.armWindowsCrossArchitectureList) {
 817             if (isBuildJob) {
 818                 affinityOptions = [ "use_arm64_build_machine" : true ]
 819             } else if (isTestJob) {
 820                 affinityOptions = [ "use_arm64_build_machine" : false ]
 821             } else if (isFlowJob) {
 822                 // For the flow jobs set the machine affinity as x64
 823                 affinityArchitecture = 'x64'
 824             }
 825         }
 826     }
 827     else {
 828         if ((architecture == 'arm64') || (architecture == 'arm')) {
 829             if (isBuildJob) {
 830                 affinityOptions = ['is_build_job': true]
 831             } else if (isFlowJob) {
 832                 affinityOptions = ['is_flow_job': true]
 833             }
 834         }
 835     }
 836
 837     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 838 }
 839
 840 def static isGCStressRelatedTesting(def scenario) {
 841     // The 'r2r_gcstress15' scenario is a basic scenario.
 842     // Detect it and make it a GCStress related.
 843     if (scenario == 'r2r_gcstress15')
 844     {
 845         return true;
 846     }
 847
 848     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 849     def scenarioName = scenario.toLowerCase()
 850     def isGCStressTesting = false
 851     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 852         if (k in gcStressTestEnvVars) {
 853             isGCStressTesting = true;
 854         }
 855     }
 856     return isGCStressTesting
 857 }
 858
 859 def static isCoreFxScenario(def scenario) {
 860     def corefx_prefix = 'corefx_'
 861     if (scenario.length() < corefx_prefix.length()) {
 862         return false
 863     }
 864     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 865 }
 866
 867 def static isR2RBaselineScenario(def scenario) {
 868     return (scenario == 'r2r')
 869 }
 870
 871 def static isR2RStressScenario(def scenario) {
 872     return Constants.r2rStressScenarios.containsKey(scenario)
 873 }
 874
 875 def static isR2RScenario(def scenario) {
 876     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 877 }
 878
 879 def static isJitStressScenario(def scenario) {
 880     return Constants.jitStressModeScenarios.containsKey(scenario)
 881 }
 882
 883 def static isLongGc(def scenario) {
 884     return (scenario == 'longgc' || scenario == 'gcsimulator')
 885 }
 886
 887 def static isJitDiff(def scenario) {
 888     return (scenario == 'jitdiff')
 889 }
 890
 891 def static isGcReliabilityFramework(def scenario) {
 892     return (scenario == 'gc_reliability_framework')
 893 }
 894
 895 def static isArmWindowsScenario(def scenario) {
 896     return Constants.validArmWindowsScenarios.containsKey(scenario)
 897 }
 898
 899 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 900     if (isBuildOnly == true) {
 901         os = 'Windows_NT_BuildOnly'
 902     }
 903
 904     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 905     if (validOsPrTriggerArchConfigs != null) {
 906         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 907         if (validOsPrTriggerConfigs != null) {
 908             if (configuration in validOsPrTriggerConfigs) {
 909                 return true
 910             }
 911         }
 912     }
 913
 914     return false
 915 }
 916
 917 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 918 // scheduled with a default PR trigger despite the correlation being true at the moment.
 919 def static isInnerloopTestScenario(def scenario) {
 920     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 921 }
 922
 923 def static isCrossGenComparisonScenario(def scenario) {
 924     return (scenario == 'crossgen_comparison')
 925 }
 926
 927 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 928     assert isCrossGenComparisonScenario(scenario)
 929     return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
 930 }
 931
 932 def static getFxBranch(def branch) {
 933     def fxBranch = branch
 934     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 935     // branch even though CoreFX doesn't have such a branch.
 936     if (branch == 'dev/unix_test_workflow') {
 937         fxBranch = 'master'
 938     }
 939     return fxBranch
 940 }
 941
 942 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 943     // 2 hours (120 minutes) is the default timeout
 944     def timeout = 120
 945
 946     if (!isInnerloopTestScenario(scenario)) {
 947         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 948         timeout = 240
 949     }
 950
 951     if (!isBuildOnly) {
 952         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 953         if (isGCStressRelatedTesting(scenario)) {
 954             timeout = 4320
 955         }
 956         else if (isCoreFxScenario(scenario)) {
 957             timeout = 360
 958         }
 959         else if (isJitStressScenario(scenario)) {
 960             timeout = 300
 961         }
 962         else if (isR2RBaselineScenario(scenario)) {
 963             timeout = 240
 964         }
 965         else if (isLongGc(scenario)) {
 966             timeout = 1440
 967         }
 968         else if (isJitDiff(scenario)) {
 969             timeout = 240
 970         }
 971         else if (isGcReliabilityFramework(scenario)) {
 972             timeout = 1440
 973         }
 974         else if (architecture == 'armem' || architecture == 'arm64') {
 975             timeout = 240
 976         }
 977
 978         if (architecture == 'arm') {
 979             // ARM32 machines are particularly slow.
 980             timeout += 120
 981         }
 982     }
 983
 984     if (configuration == 'Debug') {
 985         // Debug runs can be very slow. Add an hour.
 986         timeout += 60
 987     }
 988
 989     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 990         // AltJit runs compile all methods twice.
 991         timeout *= 2
 992     }
 993
 994     // If we've changed the timeout from the default, set it in the job.
 995
 996     if (timeout != 120) {
 997         Utilities.setJobTimeout(newJob, timeout)
 998     }
 999 }
1000
1001 def static getJobFolder(def scenario) {
1002     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1003         return 'jitstress'
1004     }
1005     if (scenario == 'illink') {
1006         return 'illink'
1007     }
1008     return ''
1009 }
1010
1011 def static getStressModeDisplayName(def scenario) {
1012     def displayStr = ''
1013     Constants.jitStressModeScenarios[scenario].each{ k, v ->
1014         def prefixLength = 'COMPlus_'.length()
1015         if (k.length() >= prefixLength) {
1016             def modeName = k.substring(prefixLength, k.length())
1017             if (displayStr != '') {
1018                 // Separate multiple variables with a space.
1019                 displayStr += ' '
1020             }
1021             displayStr += modeName + '=' + v
1022         }
1023     }
1024
1025     if (isCoreFxScenario(scenario)) {
1026         displayStr = ('CoreFx ' + displayStr).trim()
1027     }
1028
1029     return displayStr
1030 }
1031
1032 def static getR2RDisplayName(def scenario) {
1033     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1034     def displayStr = scenario
1035     def prefixLength = 'r2r_'.length()
1036     if (displayStr.length() >= prefixLength) {
1037         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1038     } else if (scenario == 'r2r') {
1039         displayStr = "R2R"
1040     }
1041     return displayStr
1042 }
1043
1044 def static getScenarioDisplayString(def scenario) {
1045     switch (scenario) {
1046         case 'innerloop':
1047             return "Innerloop Build and Test"
1048
1049         case 'no_tiered_compilation_innerloop':
1050             def displayStr = getStressModeDisplayName(scenario)
1051             return "Innerloop Build and Test (Jit - ${displayStr})"
1052
1053         case 'corefx_innerloop':
1054             return "CoreFX Tests"
1055
1056         case 'normal':
1057             return "Build and Test"
1058
1059         case 'jitdiff':
1060             return "Jit Diff Build and Test"
1061
1062         case 'ilrt':
1063             return "IL RoundTrip Build and Test"
1064
1065         case 'longgc':
1066             return "Long-Running GC Build & Test"
1067
1068         case 'gcsimulator':
1069             return "GC Simulator"
1070
1071         case 'standalone_gc':
1072             return "Standalone GC"
1073
1074         case 'gc_reliability_framework':
1075             return "GC Reliability Framework"
1076
1077         case 'illink':
1078             return "via ILLink"
1079
1080         default:
1081             if (isJitStressScenario(scenario)) {
1082                 def displayStr = getStressModeDisplayName(scenario)
1083                 return "Build and Test (Jit - ${displayStr})"
1084             }
1085             else if (isR2RScenario(scenario)) {
1086                 def displayStr = getR2RDisplayName(scenario)
1087                 return "${displayStr} Build and Test"
1088             }
1089             else {
1090                 return "${scenario}"
1091             }
1092             break
1093     }
1094
1095     println("Unknown scenario: ${scenario}");
1096     assert false
1097 }
1098
1099 //
1100 // Functions to create an environment script.
1101 //      envScriptCreate -- initialize the script (call first)
1102 //      envScriptFinalize -- finalize the script (call last)
1103 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1104 //      envScriptAppendExistingScript -- append an existing script to the generated script
1105 //
1106 // Each script returns a string of commands. Concatenate all the strings together before
1107 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1108 //
1109
1110 // Initialize the environment setting script.
1111 def static envScriptCreate(def os, def stepScriptLocation) {
1112     def stepScript = ''
1113     if (os == 'Windows_NT') {
1114         stepScript += "echo Creating TestEnv script\r\n"
1115         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1116
1117         // Create at least an empty script.
1118         stepScript += "echo. > ${stepScriptLocation}\r\n"
1119     }
1120     else {
1121         stepScript += "echo Creating environment setting script\n"
1122         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1123     }
1124
1125     return stepScript
1126 }
1127
1128 // Generates the string for setting stress mode variables.
1129 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1130     def stepScript = ''
1131     if (os == 'Windows_NT') {
1132         stressModeVars.each{ k, v ->
1133             // Write out what we are writing to the script file
1134             stepScript += "echo Setting ${k}=${v}\r\n"
1135             // Write out the set itself to the script file`
1136             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1137         }
1138     }
1139     else {
1140         stressModeVars.each{ k, v ->
1141             // Write out what we are writing to the script file
1142             stepScript += "echo Setting ${k}=${v}\n"
1143             // Write out the set itself to the script file`
1144             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1145         }
1146     }
1147
1148     return stepScript
1149 }
1150
1151 // Append an existing script to an environment script.
1152 // Returns string of commands to do this.
1153 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1154     assert (os == 'Windows_NT')
1155     def stepScript = ''
1156
1157     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1158     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1159
1160     return stepScript
1161 }
1162
1163 // Finalize an environment setting script.
1164 // Returns string of commands to do this.
1165 def static envScriptFinalize(def os, def stepScriptLocation) {
1166     def stepScript = ''
1167
1168     if (os == 'Windows_NT') {
1169         // Display the resulting script. This is useful when looking at the output log file.
1170         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1171         stepScript += "type ${stepScriptLocation}\r\n"
1172     }
1173     else {
1174         stepScript += "chmod +x ${stepScriptLocation}\n"
1175     }
1176
1177     return stepScript
1178 }
1179
1180 def static isNeedDocker(def architecture, def os, def isBuild) {
1181     if (isBuild) {
1182         if (architecture == 'x86' && os == 'Ubuntu') {
1183             return true
1184         }
1185         else if (architecture == 'armem') {
1186             return true
1187         }
1188         else if (architecture == 'arm') {
1189             if (os == 'Ubuntu') {
1190                 return true
1191             }
1192         }
1193         else if (architecture == 'arm64') {
1194             if (os == 'Ubuntu16.04') {
1195                 return true
1196             }
1197         }
1198     }
1199     else {
1200         if (architecture == 'x86' && os == 'Ubuntu') {
1201             return true
1202         }
1203     }
1204     return false
1205 }
1206
1207 def static getDockerImageName(def architecture, def os, def isBuild) {
1208     // We must change some docker private images to official later
1209     if (isBuild) {
1210         if (architecture == 'x86' && os == 'Ubuntu') {
1211             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1212         }
1213         else if (architecture == 'armem') {
1214             if (os == 'Tizen') {
1215                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1216             }
1217         }
1218         else if (architecture == 'arm') {
1219             if (os == 'Ubuntu') {
1220                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1221             }
1222         }
1223         else if (architecture == 'arm64') {
1224             if (os == 'Ubuntu16.04') {
1225                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1226             }
1227         }
1228     }
1229     else {
1230         if (architecture == 'x86' && os == 'Ubuntu') {
1231             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1232         }
1233     }
1234     println("Unknown architecture to use docker: ${architecture} ${os}");
1235     assert false
1236 }
1237
1238
1239 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1240 // and only allowing using this hardware in some specific branches.
1241 def static jobRequiresLimitedHardware(def architecture, def os) {
1242     if (architecture == 'arm') {
1243         // arm Windows and Linux hardware is limited.
1244         return true
1245     }
1246     else if (architecture == 'arm64') {
1247         // arm64 Windows and Linux hardware is limited.
1248         return true
1249     }
1250     else {
1251         return false
1252     }
1253 }
1254
1255 // Calculates the name of the build job based on some typical parameters.
1256 //
1257 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1258     // If the architecture is x64, do not add that info into the build name.
1259     // Need to change around some systems and other builds to pick up the right builds
1260     // to do that.
1261
1262     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1263     if (isBuildOnly) {
1264         suffix += '_bld'
1265     }
1266     def baseName = ''
1267     switch (architecture) {
1268         case 'x64':
1269             if (scenario == 'normal') {
1270                 // For now we leave x64 off of the name for compatibility with other jobs
1271                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1272             }
1273             else if (scenario == 'formatting') {
1274                 // we don't care about the configuration for the formatting job. It runs all configs
1275                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1276             }
1277             else {
1278                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1279             }
1280             break
1281         case 'armem':
1282             // These are cross builds
1283             assert os == 'Tizen'
1284             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1285             break
1286         case 'arm':
1287         case 'arm64':
1288             // These are cross builds
1289             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1290             break
1291         case 'x86':
1292         case 'x86_arm_altjit':
1293         case 'x64_arm64_altjit':
1294             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1295             break
1296         default:
1297             println("Unknown architecture: ${architecture}");
1298             assert false
1299             break
1300     }
1301
1302     return baseName + suffix
1303 }
1304
1305 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1306     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1307
1308     // Limited hardware is restricted for non-PR triggers to certain branches.
1309     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1310         return
1311     }
1312
1313     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1314     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1315     if (architecture == 'x86' && os == 'Ubuntu') {
1316         return
1317     }
1318
1319     // Check scenario.
1320     switch (scenario) {
1321         case 'innerloop':
1322         case 'no_tiered_compilation_innerloop':
1323             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1324             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1325             // We should never get here (in the "innerloop cases) for non-PR jobs, except for this TEMPORARY exception.
1326             // Only trigger the flow job, not the build job.
1327             assert (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked'))
1328             if (isFlowJob) {
1329                 addGithubPushTriggerHelper(job)
1330             }
1331             break
1332
1333         case 'crossgen_comparison':
1334             break
1335         case 'normal':
1336             switch (architecture) {
1337                 case 'x64':
1338                 case 'x86':
1339                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1340                         addPeriodicTriggerHelper(job, '@daily')
1341                     }
1342                     else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1343                         addGithubPushTriggerHelper(job)
1344                     }
1345                     break
1346                 case 'arm64':
1347                     if (os == 'Windows_NT') {
1348                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1349                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1350                             // Do the builds daily.
1351                             addPeriodicTriggerHelper(job, '@daily')
1352                         }
1353                     }
1354                     else {
1355                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1356                         if (isFlowJob) {
1357                             addPeriodicTriggerHelper(job, '@daily')
1358                         }
1359                     }
1360                     break
1361                 case 'arm':
1362                     if (os == 'Windows_NT') {
1363                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1364                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1365                             // Do the builds daily.
1366                             addPeriodicTriggerHelper(job, '@daily')
1367                         }
1368                     }
1369                     else {
1370                         assert os == 'Ubuntu'
1371                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1372                         if (isFlowJob) {
1373                             // Currently no push triggers, with limited arm Linux hardware.
1374                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1375                             addPeriodicTriggerHelper(job, '@daily')
1376                         }
1377                     }
1378                     break
1379                 case 'armem':
1380                     addGithubPushTriggerHelper(job)
1381                     break
1382                 case 'x86_arm_altjit':
1383                 case 'x64_arm64_altjit':
1384                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1385                     if (configuration == 'Checked') {
1386                         addGithubPushTriggerHelper(job)
1387                     }
1388                     break
1389                 default:
1390                     println("Unknown architecture: ${architecture}");
1391                     assert false
1392                     break
1393             }
1394             break
1395         case 'r2r':
1396             assert !(os in bidailyCrossList)
1397             // r2r gets a push trigger for checked/release
1398             if (configuration == 'Checked' || configuration == 'Release') {
1399                 if (architecture == 'x64' && os != 'OSX10.12') {
1400                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1401                     if (isFlowJob || os == 'Windows_NT') {
1402                         addGithubPushTriggerHelper(job)
1403                     }
1404                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1405                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1406                     if (isFlowJob) {
1407                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1408                     }
1409                 }
1410                 // For x86, only add per-commit jobs for Windows
1411                 else if (architecture == 'x86') {
1412                     if (os == 'Windows_NT') {
1413                         addGithubPushTriggerHelper(job)
1414                     }
1415                 }
1416                 // arm r2r jobs should only run weekly.
1417                 else if (architecture == 'arm') {
1418                     if (isFlowJob) {
1419                         addPeriodicTriggerHelper(job, '@weekly')
1420                     }
1421                 }
1422                 // arm64 r2r jobs should only run weekly.
1423                 else if (architecture == 'arm64') {
1424                     if (isFlowJob) {
1425                         addPeriodicTriggerHelper(job, '@weekly')
1426                     }
1427                 }
1428             }
1429             break
1430         case 'r2r_jitstress1':
1431         case 'r2r_jitstress2':
1432         case 'r2r_jitstress1_tiered':
1433         case 'r2r_jitstress2_tiered':
1434         case 'r2r_jitstressregs1':
1435         case 'r2r_jitstressregs2':
1436         case 'r2r_jitstressregs3':
1437         case 'r2r_jitstressregs4':
1438         case 'r2r_jitstressregs8':
1439         case 'r2r_jitstressregs0x10':
1440         case 'r2r_jitstressregs0x80':
1441         case 'r2r_jitstressregs0x1000':
1442         case 'r2r_jitminopts':
1443         case 'r2r_jitforcerelocs':
1444         case 'r2r_gcstress15':
1445         case 'r2r_no_tiered_compilation':
1446             assert !(os in bidailyCrossList)
1447
1448             // GCStress=C is currently not supported on OS X
1449             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1450                 break
1451             }
1452
1453             if (configuration == 'Checked' || configuration == 'Release') {
1454                 if (architecture == 'x64') {
1455                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1456                     if (isFlowJob || os == 'Windows_NT') {
1457                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1458                     }
1459                 }
1460                 // For x86, only add periodic jobs for Windows
1461                 else if (architecture == 'x86') {
1462                     if (os == 'Windows_NT') {
1463                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1464                     }
1465                 }
1466                 else if (architecture == 'arm') {
1467                     if (isFlowJob) {
1468                         addPeriodicTriggerHelper(job, '@weekly')
1469                     }
1470                 }
1471                 else if (architecture == 'arm64') {
1472                     if (isFlowJob) {
1473                         addPeriodicTriggerHelper(job, '@weekly')
1474                     }
1475                 }
1476             }
1477             break
1478         case 'longgc':
1479             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1480             assert configuration == 'Release'
1481             assert architecture == 'x64'
1482             addPeriodicTriggerHelper(job, '@daily')
1483             // TODO: Add once external email sending is available again
1484             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1485             break
1486         case 'gcsimulator':
1487             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1488             assert configuration == 'Release'
1489             assert architecture == 'x64'
1490             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1491             // TODO: Add once external email sending is available again
1492             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1493             break
1494         case 'standalone_gc':
1495             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1496             assert (configuration == 'Release' || configuration == 'Checked')
1497             // TODO: Add once external email sending is available again
1498             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1499             addPeriodicTriggerHelper(job, '@daily')
1500             break
1501         case 'gc_reliability_framework':
1502             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1503             assert (configuration == 'Release' || configuration == 'Checked')
1504             // Only triggered by phrase.
1505             break
1506         case 'ilrt':
1507             assert !(os in bidailyCrossList)
1508             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1509             if (architecture == 'x64' && configuration == 'Release') {
1510                 if (isFlowJob || os == 'Windows_NT') {
1511                     addPeriodicTriggerHelper(job, '@daily')
1512                 }
1513             }
1514             break
1515         case 'jitdiff':
1516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1517             assert configuration == 'Checked'
1518             assert (architecture == 'x64' || architecture == 'x86')
1519             addGithubPushTriggerHelper(job)
1520             break
1521         case 'formatting':
1522             assert (os == 'Windows_NT' || os == "Ubuntu")
1523             assert architecture == 'x64'
1524             addGithubPushTriggerHelper(job)
1525             break
1526         case 'jitstressregs1':
1527         case 'jitstressregs2':
1528         case 'jitstressregs3':
1529         case 'jitstressregs4':
1530         case 'jitstressregs8':
1531         case 'jitstressregs0x10':
1532         case 'jitstressregs0x80':
1533         case 'jitstressregs0x1000':
1534         case 'minopts':
1535         case 'tieredcompilation':
1536         case 'no_tiered_compilation':
1537         case 'forcerelocs':
1538         case 'jitstress1':
1539         case 'jitstress2':
1540         case 'jitstress1_tiered':
1541         case 'jitstress2_tiered':
1542         case 'jitstress2_jitstressregs1':
1543         case 'jitstress2_jitstressregs2':
1544         case 'jitstress2_jitstressregs3':
1545         case 'jitstress2_jitstressregs4':
1546         case 'jitstress2_jitstressregs8':
1547         case 'jitstress2_jitstressregs0x10':
1548         case 'jitstress2_jitstressregs0x80':
1549         case 'jitstress2_jitstressregs0x1000':
1550         case 'tailcallstress':
1551         case 'jitsse2only':
1552         case 'jitnosimd':
1553         case 'jitnox86hwintrinsic':
1554         case 'jitincompletehwintrinsic':
1555         case 'jitx86hwintrinsicnoavx':
1556         case 'jitx86hwintrinsicnoavx2':
1557         case 'jitx86hwintrinsicnosimd':
1558         case 'corefx_baseline':
1559         case 'corefx_minopts':
1560         case 'corefx_tieredcompilation':
1561         case 'corefx_jitstress1':
1562         case 'corefx_jitstress2':
1563         case 'corefx_jitstressregs1':
1564         case 'corefx_jitstressregs2':
1565         case 'corefx_jitstressregs3':
1566         case 'corefx_jitstressregs4':
1567         case 'corefx_jitstressregs8':
1568         case 'corefx_jitstressregs0x10':
1569         case 'corefx_jitstressregs0x80':
1570         case 'corefx_jitstressregs0x1000':
1571         case 'zapdisable':
1572             if (os == 'CentOS7.1') {
1573                 break
1574             }
1575             if (os in bidailyCrossList) {
1576                 break
1577             }
1578             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1579             // build for the flow job. We don't need cron jobs for these. Note that the
1580             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1581             // function very early, so only non-Windows gets here.
1582             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1583                 break
1584             }
1585             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1586                 break
1587             }
1588             if (jobRequiresLimitedHardware(architecture, os)) {
1589                 if ((architecture == 'arm64') && (os == 'Ubuntu16.04')) {
1590                     // These jobs are very fast on Linux/arm64 hardware, so run them daily.
1591                     addPeriodicTriggerHelper(job, '@daily')
1592                 }
1593                 else {
1594                     addPeriodicTriggerHelper(job, '@weekly')
1595                 }
1596             }
1597             else {
1598                 addPeriodicTriggerHelper(job, '@daily')
1599             }
1600             break
1601         case 'heapverify1':
1602         case 'gcstress0x3':
1603             if (os == 'CentOS7.1') {
1604                 break
1605             }
1606             if (os in bidailyCrossList) {
1607                 break
1608             }
1609             addPeriodicTriggerHelper(job, '@weekly')
1610             break
1611         case 'gcstress0xc':
1612         case 'gcstress0xc_zapdisable':
1613         case 'gcstress0xc_zapdisable_jitstress2':
1614         case 'gcstress0xc_zapdisable_heapverify1':
1615         case 'gcstress0xc_jitstress1':
1616         case 'gcstress0xc_jitstress2':
1617         case 'gcstress0xc_minopts_heapverify1':
1618             if (os == 'OSX10.12') {
1619                 // GCStress=C is currently not supported on OS X
1620                 break
1621             }
1622             if (os == 'CentOS7.1') {
1623                 break
1624             }
1625             if (os in bidailyCrossList) {
1626                 break
1627             }
1628             addPeriodicTriggerHelper(job, '@weekly')
1629             break
1630
1631         case 'illink':
1632             // Testing on other operating systems TBD
1633             assert (os == 'Windows_NT' || os == 'Ubuntu')
1634             if (architecture == 'x64' || architecture == 'x86') {
1635                 if (configuration == 'Checked') {
1636                     addPeriodicTriggerHelper(job, '@daily')
1637                 }
1638             }
1639             break
1640
1641         default:
1642             println("Unknown scenario: ${scenario}");
1643             assert false
1644             break
1645     }
1646     return
1647 }
1648
1649 // **************************
1650 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1651 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1652 // builds will do a couple extra steps.
1653 // **************************
1654
1655 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1656 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1657 // it's just used for internal builds
1658 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1659 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1660     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1661
1662     if (isWindowsBuildOnlyJob) {
1663         return
1664     }
1665
1666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1667     // Non pull request builds.
1668     if (!isPR) {
1669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1670         return
1671     }
1672
1673     def arm64Users = [
1674         'adityamandaleeka',
1675         'AndyAyersMS',
1676         'briansull',
1677         'BruceForstall',
1678         'CarolEidt',
1679         'davidwrighton',
1680         'echesakovMSFT',
1681         'erozenfeld',
1682         'janvorli',
1683         'jashook',
1684         'pgodeq',
1685         'RussKeldorph',
1686         'sandreenko',
1687         'sdmaclea',
1688         'swaroop-sridhar',
1689         'jkotas',
1690         'markwilkie',
1691         'weshaggard'
1692     ]
1693
1694     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1695     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1696
1697     def contextString = ""
1698     def triggerString = ""
1699     def needsTrigger = true
1700     def isDefaultTrigger = false
1701     def isArm64PrivateJob = false
1702     def scenarioString = ""
1703
1704     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1705     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1706     // some jobs where the Debug configuration just does builds, no tests.
1707     //
1708     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1709     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1710     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1711     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1712
1713     if (architecture == 'armem') {
1714         assert os == 'Tizen'
1715         architecture = 'armel'
1716     }
1717
1718     switch (architecture) {
1719         case 'x64_arm64_altjit':
1720         case 'x86_arm_altjit':
1721             // TODO: for consistency, add "Build and Test" at end.
1722             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1723             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1724             break
1725
1726         case 'armel':
1727         case 'arm':
1728         case 'arm64':
1729             contextString = "${os} ${architecture} Cross ${configuration}"
1730             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1731
1732             if (scenario == 'innerloop') {
1733                 contextString += " Innerloop"
1734                 triggerString += "\\W+Innerloop"
1735             }
1736             else {
1737                 contextString += " ${scenario}"
1738                 triggerString += "\\W+${scenario}"
1739             }
1740
1741             if (configuration == 'Debug') {
1742                 contextString += " Build"
1743                 triggerString += "\\W+Build"
1744             }
1745             else {
1746                 contextString += " Build and Test"
1747                 triggerString += "\\W+Build and Test"
1748             }
1749
1750             triggerString += ".*"
1751             break
1752
1753         default:
1754             scenarioString = getScenarioDisplayString(scenario)
1755             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1756             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1757
1758             switch (scenario) {
1759                 case 'normal':
1760                     triggerString += "\\W+Build and Test.*"
1761                     break
1762
1763                 case 'corefx_innerloop': // maintain this asymmetry
1764                     triggerString += "\\W+CoreFX Tests.*"
1765                     break
1766
1767                 default:
1768                     triggerString += "\\W+${scenario}.*"
1769                     break
1770             }
1771
1772             triggerString += ".*"
1773             break
1774     }
1775
1776     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1777     // non-flow jobs that are only used as part of flow jobs.
1778
1779     switch (architecture) {
1780         case 'x64': // editor brace matching: {
1781             if (scenario == 'formatting') {
1782                 assert configuration == 'Checked'
1783                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1784                     isDefaultTrigger = true
1785                     contextString = "${os} ${architecture} Formatting"
1786                 }
1787                 break
1788             }
1789
1790             switch (os) {
1791                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1792                 case 'Debian8.4':
1793                 case 'RHEL7.2':
1794                     if (scenario == 'innerloop') {
1795                         assert !isFlowJob
1796                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1797                         isDefaultTrigger = true
1798                         break
1799                     }
1800
1801                     // fall through
1802
1803                 case 'Fedora24':
1804                 case 'Ubuntu16.04':
1805                 case 'Ubuntu16.10':
1806                     assert !isFlowJob
1807                     assert scenario != 'innerloop'
1808                     contextString = "${os} ${architecture} ${configuration} Build"
1809                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1810                     break
1811
1812                 case 'Ubuntu':
1813                     if (scenario == 'illink') {
1814                         break
1815                     }
1816                     else if (scenario == 'corefx_innerloop') {
1817                         if (configuration == 'Checked') {
1818                             isDefaultTrigger = true
1819                         }
1820                         break
1821                     }
1822
1823                     // fall through
1824
1825                 case 'OSX10.12':
1826                     // Triggers on the non-flow jobs aren't necessary here
1827                     // Corefx testing uses non-flow jobs.
1828                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1829                         needsTrigger = false
1830                         break
1831                     }
1832                     switch (scenario) {
1833                         case 'innerloop':
1834                             isDefaultTrigger = true
1835                             break
1836
1837                         case 'no_tiered_compilation_innerloop':
1838                             if (os == 'Ubuntu') {
1839                                 isDefaultTrigger = true
1840                             }
1841                             break
1842
1843                         default:
1844                             break
1845                     }
1846                     break
1847
1848                 case 'CentOS7.1':
1849                     switch (scenario) {
1850                         case 'innerloop':
1851                             // CentOS uses checked for default PR tests while debug is build only
1852                             if (configuration == 'Debug') {
1853                                 isDefaultTrigger = true
1854                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1855                                 break
1856                             }
1857
1858                             // Make sure this is a flow job to get build and test.
1859                             if (!isFlowJob) {
1860                                 needsTrigger = false
1861                                 break
1862                             }
1863
1864                             if (configuration == 'Checked') {
1865                                 assert job.name.contains("flow")
1866                                 isDefaultTrigger = true
1867                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1868                             }
1869                             break
1870
1871                         case 'normal':
1872                             // Make sure this is a flow job to get build and test.
1873                             if (!isFlowJob) {
1874                                 needsTrigger = false
1875                                 break
1876                             }
1877                             break
1878
1879                         default:
1880                             break
1881                     }
1882                     break
1883
1884                 case 'Windows_NT':
1885                     switch (scenario) {
1886                         case 'innerloop':
1887                         case 'no_tiered_compilation_innerloop':
1888                             isDefaultTrigger = true
1889                             break
1890
1891                         case 'corefx_innerloop':
1892                             if (configuration == 'Checked' || configuration == 'Release') {
1893                                 isDefaultTrigger = true
1894                             }
1895                             break
1896
1897                         default:
1898                             break
1899                     }
1900                     break
1901
1902                 default:
1903                     println("Unknown os: ${os}");
1904                     assert false
1905                     break
1906
1907             } // switch (os)
1908
1909             break
1910         // editor brace matching: }
1911
1912         case 'armel': // editor brace matching: {
1913             job.with {
1914                 publishers {
1915                     azureVMAgentPostBuildAction {
1916                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1917                     }
1918                 }
1919             }
1920
1921             switch (os) {
1922                 case 'Tizen':
1923                     if (scenario == 'innerloop') {
1924                         if (configuration == 'Checked') {
1925                             isDefaultTrigger = true
1926                         }
1927                     }
1928                     break
1929             }
1930
1931             break
1932         // editor brace matching: }
1933
1934         case 'arm':
1935         case 'arm64': // editor brace matching: {
1936
1937             switch (os) {
1938                 case 'Ubuntu':
1939                 case 'Ubuntu16.04':
1940
1941                     // Triggers on the non-flow jobs aren't necessary
1942                     if (!isFlowJob) {
1943                         needsTrigger = false
1944                         break
1945                     }
1946
1947                     switch (scenario) {
1948                         case 'innerloop':
1949                         case 'no_tiered_compilation_innerloop':
1950                             if (configuration == 'Checked') {
1951                                 // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1952                                 //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1953                                 if (architecture == 'arm64') {
1954                                     break
1955                                 }
1956                                 isDefaultTrigger = true
1957                             }
1958                             break
1959                     }
1960                     break
1961
1962                 case 'Windows_NT':
1963                     assert isArmWindowsScenario(scenario)
1964
1965                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1966                     // the non-flow Build job. All others need a trigger on the flow job.
1967                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1968                     if (isFlowJob != needsFlowJobTrigger) {
1969                         needsTrigger = false
1970                         break
1971                     }
1972
1973                     switch (scenario) {
1974                         case 'innerloop':
1975                             if (configuration == 'Debug') {
1976                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1977                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1978                                 // People in arm64Users will get both this and the Checked Build and Test job.
1979                                 isDefaultTrigger = true
1980                             } else if (configuration == 'Checked') {
1981                                 isDefaultTrigger = true
1982                                 isArm64PrivateJob = true
1983                             }
1984                             break
1985                         default:
1986                             isArm64PrivateJob = true
1987                             break
1988                     }
1989                     break
1990                 default:
1991                     println("NYI os: ${os}");
1992                     assert false
1993                     break
1994             }
1995             break
1996
1997         // editor brace matching: }
1998         case 'x86': // editor brace matching: {
1999             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2000             if (os == 'Ubuntu') {
2001                 // Triggers on the non-flow jobs aren't necessary here
2002                 if (!isFlowJob) {
2003                     needsTrigger = false
2004                     break
2005                 }
2006
2007                 // on-demand only for ubuntu x86
2008                 contextString = "${os} ${architecture} ${configuration} Build"
2009                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2010                 break
2011             }
2012             switch (scenario) {
2013                 case 'innerloop':
2014                 case 'no_tiered_compilation_innerloop':
2015                     isDefaultTrigger = true
2016                     break
2017                 default:
2018                     break
2019             }
2020             break
2021
2022         // editor brace matching: }
2023         case 'x64_arm64_altjit':
2024         case 'x86_arm_altjit':
2025             // Everything default
2026             break
2027
2028         default:
2029             println("Unknown architecture: ${architecture}");
2030             assert false
2031             break
2032     }
2033
2034     if (needsTrigger) {
2035         if (isArm64PrivateJob) {
2036             if (isDefaultTrigger) {
2037                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2038             }
2039             else {
2040                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2041             }
2042         }
2043         else {
2044             if (isDefaultTrigger) {
2045                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2046             }
2047             else {
2048                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2049             }
2050         }
2051     }
2052 }
2053
2054 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2055     def buildCommands = []
2056     def osGroup = getOSGroup(os)
2057     def lowerConfiguration = configuration.toLowerCase()
2058
2059     def priority = '1'
2060     if (isInnerloopTestScenario(scenario)) {
2061         priority = '0'
2062     }
2063
2064     def doCoreFxTesting = isCoreFxScenario(scenario)
2065
2066     // Calculate the build steps, archival, and xunit results
2067     switch (os) {
2068         case 'Windows_NT': // editor brace matching: {
2069             switch (architecture) {
2070                 case 'x64':
2071                 case 'x86':
2072                 case 'x86_arm_altjit':
2073                 case 'x64_arm64_altjit':
2074                     def arch = architecture
2075                     def buildOpts = ''
2076                     if (architecture == 'x86_arm_altjit') {
2077                         arch = 'x86'
2078                     }
2079                     else if (architecture == 'x64_arm64_altjit') {
2080                         arch = 'x64'
2081                     }
2082
2083                     if (scenario == 'formatting') {
2084                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2085                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2086                         break
2087                     }
2088
2089                     if (scenario == 'illink') {
2090                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2091                     }
2092
2093                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2094                     if ((lowerConfiguration == 'release') &&
2095                         (scenario in Constants.basicScenarios) &&
2096                         (architecture != 'x86_arm_altjit') &&
2097                         (architecture != 'x64_arm64_altjit')) {
2098
2099                         buildOpts += ' -enforcepgo'
2100                     }
2101
2102                     if (doCoreFxTesting) {
2103                         buildOpts += ' skiptests';
2104                     } else {
2105                         buildOpts += " -priority=${priority}"
2106                     }
2107
2108                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2109                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2110                     // 35 characters long.
2111
2112                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2113
2114                     if (!isBuildOnly) {
2115                         def runtestArguments = ''
2116                         def testOpts = 'collectdumps'
2117
2118                         if (isR2RScenario(scenario)) {
2119
2120                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2121                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2122                             // to cause the tests to be crossgen'ed.
2123
2124                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2125                                 testOpts += ' crossgenaltjit protononjit.dll'
2126                             } else {
2127                                 testOpts += ' crossgen'
2128                             }
2129
2130                             testOpts += ' runcrossgentests'
2131                         }
2132                         else if (scenario == 'jitdiff') {
2133                             testOpts += ' jitdisasm crossgen'
2134                         }
2135                         else if (scenario == 'ilrt') {
2136                             testOpts += ' ilasmroundtrip'
2137                         }
2138                         else if (isLongGc(scenario)) {
2139                             testOpts += " ${scenario} sequential"
2140                         }
2141                         else if (scenario == 'standalone_gc') {
2142                             testOpts += ' gcname clrgc.dll'
2143                         }
2144                         else if (scenario == 'illink') {
2145                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2146                         }
2147
2148                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2149                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2150                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2151                         // which is read by the xunit harness.)
2152                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2153                         {
2154                             def timeout = 1800000
2155                             testOpts += " timeout ${timeout}"
2156                         }
2157
2158                         // If we are running a stress mode, we should write out the set of key
2159                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2160
2161                         def envScriptPath = ''
2162                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2163                             def buildCommandsStr = ''
2164                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2165                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2166
2167                             if (isJitStressScenario(scenario)) {
2168                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2169                             }
2170                             else if (isR2RStressScenario(scenario)) {
2171                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2172                             }
2173
2174                             if (architecture == 'x86_arm_altjit') {
2175                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2176                             }
2177                             else if (architecture == 'x64_arm64_altjit') {
2178                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2179                             }
2180
2181                             envScriptFinalize(os, envScriptPath)
2182
2183                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2184                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2185                             // into a single script.
2186                             buildCommands += buildCommandsStr
2187                         }
2188                         else if (architecture == 'x86_arm_altjit') {
2189                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2190                         }
2191                         else if (architecture == 'x64_arm64_altjit') {
2192                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2193                         }
2194                         if (envScriptPath != '') {
2195                             testOpts += " TestEnv ${envScriptPath}"
2196                         }
2197
2198                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2199
2200                         if (doCoreFxTesting) {
2201                             if (scenario == 'corefx_innerloop') {
2202                                 // Create CORE_ROOT and testhost
2203                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2204                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2205
2206                                 // Archive and process (only) the test results
2207                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2208                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2209                             }
2210                             else {
2211                                 def workspaceRelativeFxRoot = "_/fx"
2212                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2213                                 def fxBranch = getFxBranch(branch)
2214
2215                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2216
2217                                 // Archive and process (only) the test results
2218                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2219                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2220
2221                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2222                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2223                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2224                             }
2225                         }
2226                         else if (isGcReliabilityFramework(scenario)) {
2227                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2228                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2229                         }
2230                         else {
2231                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2232                         }
2233                     } // end if (!isBuildOnly)
2234
2235                     if (!doCoreFxTesting) {
2236                         // Run the rest of the build
2237                         // Build the mscorlib for the other OS's
2238                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2239                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2240
2241                         if (arch == 'x64') {
2242                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2243                         }
2244
2245                         // Zip up the tests directory so that we don't use so much space/time copying
2246                         // 10s of thousands of files around.
2247                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2248
2249                         if (!isJitStressScenario(scenario)) {
2250                             // For Windows, pull full test results and test drops for x86/x64.
2251                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2252                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2253                         }
2254
2255                         if (scenario == 'jitdiff') {
2256                             // retrieve jit-dasm output for base commit, and run jit-diff
2257                             if (!isBuildOnly) {
2258                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2259                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2260                             }
2261                         }
2262
2263                         if (!isBuildOnly) {
2264                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2265                         }
2266                     }
2267                     break
2268                 case 'arm':
2269                 case 'arm64':
2270                     assert isArmWindowsScenario(scenario)
2271
2272                     def buildOpts = ''
2273
2274                     if (doCoreFxTesting) {
2275                         buildOpts += ' skiptests'
2276                     } else {
2277                         buildOpts += " -priority=${priority}"
2278                     }
2279
2280                     // This is now a build only job. Do not run tests. Use the flow job.
2281                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2282
2283                     if (doCoreFxTesting) {
2284                         assert isBuildOnly
2285
2286                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2287                         def envScriptPath = ''
2288                         def buildCommandsStr = ''
2289                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2290                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2291                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2292                         envScriptFinalize(os, envScriptPath)
2293                         buildCommands += buildCommandsStr
2294
2295                         def workspaceRelativeFxRootLinux = "_/fx"
2296                         def workspaceRelativeFxRootWin = "_\\fx"
2297                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2298                         def fxBranch = getFxBranch(branch)
2299
2300                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2301
2302                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2303                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2304                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2305
2306                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2307                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2308                     } else {
2309                         // Zip up the tests directory so that we don't use so much space/time copying
2310                         // 10s of thousands of files around.
2311                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2312
2313                         // Add archival.
2314                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2315                     }
2316                     break
2317                 default:
2318                     println("Unknown architecture: ${architecture}");
2319                     assert false
2320                     break
2321             }
2322             break
2323         // end case 'Windows_NT'; editor brace matching: }
2324         case 'Ubuntu':
2325         case 'Ubuntu16.04':
2326         case 'Ubuntu16.10':
2327         case 'Debian8.4':
2328         case 'OSX10.12':
2329         case 'CentOS7.1':
2330         case 'RHEL7.2':
2331         case 'Tizen':
2332         case 'Fedora24': // editor brace matching: {
2333             switch (architecture) {
2334                 case 'x64':
2335                 case 'x86':
2336                     if (architecture == 'x86' && os == 'Ubuntu') {
2337                         // build and PAL test
2338                         def dockerImage = getDockerImageName(architecture, os, true)
2339                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2340                         dockerImage = getDockerImageName(architecture, os, false)
2341                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2342                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2343                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2344                         break
2345                     }
2346
2347                     if (scenario == 'formatting') {
2348                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2349                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2350                         break
2351                     }
2352
2353                     if (scenario == 'illink') {
2354                         assert(os == 'Ubuntu')
2355                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2356                     }
2357
2358                     if (!doCoreFxTesting) {
2359                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2360                         // only on supported OS platforms.
2361                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2362                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2363
2364                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2365                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2366
2367                         // Basic archiving of the build
2368                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2369                         // And pal tests
2370                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2371                     }
2372                     else {
2373                         if (scenario == 'corefx_innerloop') {
2374                             assert os == 'Ubuntu' || 'OSX10.12'
2375                             assert architecture == 'x64'
2376
2377                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2378                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2379                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2380
2381                             break
2382                             // Archive and process (only) the test results
2383                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2384                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2385                         }
2386                         else {
2387                             // Corefx stress testing
2388                             assert os == 'Ubuntu'
2389                             assert architecture == 'x64'
2390                             assert lowerConfiguration == 'checked'
2391                             assert isJitStressScenario(scenario)
2392
2393                             // Build coreclr
2394                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2395
2396                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2397
2398                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2399                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2400                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2401                             buildCommands += envScriptCmds
2402
2403                             // Build and text corefx
2404                             def workspaceRelativeFxRoot = "_/fx"
2405                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2406                             def fxBranch = getFxBranch(branch)
2407
2408                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2409
2410                             // Archive and process (only) the test results
2411                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2412                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2413                         }
2414                     }
2415                     break
2416                 case 'armem':
2417                     // Emulator cross builds for ARM runs on Tizen currently
2418                     assert os == 'Tizen'
2419
2420                     def arm_abi = "armel"
2421                     def linuxCodeName = "tizen"
2422
2423                     // Unzip the Windows test binaries first. Exit with 0
2424                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2425
2426                     // Unpack the corefx binaries
2427                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2428                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2429
2430                     // Call the ARM CI script to cross build and test using docker
2431                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2432                     --mode=docker \\
2433                     --${arm_abi} \\
2434                     --linuxCodeName=${linuxCodeName} \\
2435                     --buildConfig=${lowerConfiguration} \\
2436                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2437                     --coreFxBinDir=./bin/CoreFxBinDir \\
2438                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2439
2440                     // Basic archiving of the build, no pal tests
2441                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2442                     break
2443                 case 'arm64':
2444                 case 'arm':
2445                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2446                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2447
2448                     // Add some useful information to the log file. Ignore return codes.
2449                     buildCommands += "uname -a || true"
2450
2451                     def additionalOpts = ""
2452                     if (architecture == 'arm') {
2453                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2454                     }
2455
2456                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2457                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2458                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2459                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2460                     // ZIP up the generated CoreFX runtime and tests.
2461
2462                     def dockerImage = getDockerImageName(architecture, os, true)
2463                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2464
2465                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2466
2467                     if (doCoreFxTesting) {
2468                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2469
2470                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2471                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2472                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2473                         buildCommands += envScriptCmds
2474
2475                         // Build and text corefx
2476                         def workspaceRelativeFxRootLinux = "_/fx"
2477                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2478                         def fxBranch = getFxBranch(branch)
2479
2480                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2481
2482                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2483                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2484                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2485
2486                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2487                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2488                     }
2489                     else if (isCrossGenComparisonScenario(scenario)) {
2490                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2491
2492                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2493                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2494                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2495                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2496                         def workspaceRelativeResultsDir = "_"
2497                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2498                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2499                         getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
2500                             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2501                             def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2502
2503                             buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2504                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2505                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2506                         } // crossArch
2507                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2508                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2509                     }
2510                     else if (architecture == 'arm') {
2511                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2512                         // download the appropriate CoreFX packages.
2513                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2514                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2515
2516                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2517
2518                         // ZIP up for the test job (created in the flow job code):
2519                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2520                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2521                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2522                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2523
2524                         // These commands are assumed to be run from the root of the workspace.
2525                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2526                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2527
2528                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2529                     }
2530                     else {
2531                         assert architecture == 'arm64'
2532
2533                         // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2534                         // Linux/arm64 does not use Windows-built tests.
2535
2536                         def testBuildOpts = ""
2537                         if (priority == '1') {
2538                             testBuildOpts = "priority1"
2539                         }
2540
2541                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2542
2543                         // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2544                         buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
2545
2546                         // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
2547                         // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
2548                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2549
2550                         Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2551                     }
2552
2553                     // Archive the build logs from both product and test builds.
2554                     Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
2555
2556                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2557                     // the cleanup task in Jenkins can't remove.
2558                     newJob.with {
2559                         publishers {
2560                             azureVMAgentPostBuildAction {
2561                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2562                             }
2563                         }
2564                     }
2565                     break
2566                 default:
2567                     println("Unknown architecture: ${architecture}");
2568                     assert false
2569                     break
2570             }
2571             break
2572         // editor brace matching: }
2573         default:
2574             println("Unknown os: ${os}");
2575             assert false
2576             break
2577     } // os
2578
2579     return buildCommands
2580 }
2581
2582 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2583 // Returns true if the job should be generated.
2584 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2585 {
2586     // The various "innerloop" jobs are only available as PR triggered.
2587
2588     if (!isPR) {
2589         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
2590             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
2591             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
2592             return true
2593         }
2594
2595         if (isInnerloopTestScenario(scenario)) {
2596             return false
2597         }
2598
2599         if (scenario == 'corefx_innerloop') {
2600             return false
2601         }
2602     }
2603
2604     // Tizen is only supported for armem architecture
2605     if (os == 'Tizen' && architecture != 'armem') {
2606         return false
2607     }
2608
2609     // Filter based on architecture.
2610
2611     switch (architecture) {
2612         case 'arm':
2613             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2614                 return false
2615             }
2616             break
2617         case 'arm64':
2618             if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2619                 return false
2620             }
2621             break
2622         case 'armem':
2623             if (os != 'Tizen') {
2624                 return false
2625             }
2626             break
2627         case 'x86_arm_altjit':
2628         case 'x64_arm64_altjit':
2629             if (os != 'Windows_NT') {
2630                 return false
2631             }
2632             break
2633         case 'x86':
2634             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2635                 return false
2636             }
2637             break
2638         case 'x64':
2639             // Everything implemented
2640             break
2641         default:
2642             println("Unknown architecture: ${architecture}")
2643             assert false
2644             break
2645     }
2646
2647     // Which (Windows) build only jobs are required?
2648
2649     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2650
2651     if (isBuildOnly) {
2652         switch (architecture) {
2653             case 'arm':
2654             case 'arm64':
2655                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2656                 if (!isCoreFxScenario(scenario)) {
2657                     return false
2658                 }
2659                 break
2660             case 'x64':
2661             case 'x86':
2662                 if (!isNormalOrInnerloop) {
2663                     return false
2664                 }
2665                 break
2666             default:
2667                 return false
2668         }
2669     }
2670
2671     // Filter based on scenario.
2672
2673     if (isJitStressScenario(scenario)) {
2674         if (configuration != 'Checked') {
2675             return false
2676         }
2677
2678         def isEnabledOS = (os == 'Windows_NT') ||
2679                           (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2680                           (os == 'Ubuntu' && architecture == 'arm') ||
2681                           (os == 'Ubuntu16.04' && architecture == 'arm64')
2682         if (!isEnabledOS) {
2683             return false
2684         }
2685
2686         switch (architecture) {
2687             case 'x64':
2688             case 'x86_arm_altjit':
2689             case 'x64_arm64_altjit':
2690                 break
2691
2692             case 'x86':
2693                 // x86 ubuntu: no stress modes
2694                 if (os == 'Ubuntu') {
2695                     return false
2696                 }
2697                 break
2698
2699             case 'arm':
2700             case 'arm64':
2701                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2702                 // No "regular" Windows arm corefx jobs, e.g.
2703                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2704                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2705                 if (os == 'Windows_NT') {
2706                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2707                         return false
2708                     }
2709                 }
2710                 else {
2711                     if (!isCoreFxScenario(scenario)) {
2712                         return false
2713                     }
2714                 }
2715                 break
2716
2717             default:
2718                 // armem: no stress jobs for ARM emulator.
2719                 return false
2720         }
2721     }
2722     else if (isR2RScenario(scenario)) {
2723         if (os != 'Windows_NT') {
2724             return false
2725         }
2726
2727         if (isR2RBaselineScenario(scenario)) {
2728             // no need for Debug scenario; Checked is sufficient
2729             if (configuration != 'Checked' && configuration != 'Release') {
2730                 return false
2731             }
2732         }
2733         else if (isR2RStressScenario(scenario)) {
2734             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2735             if (configuration != 'Checked') {
2736                 return false
2737             }
2738         }
2739
2740         switch (architecture) {
2741             case 'arm':
2742             case 'arm64':
2743                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2744                 return false
2745
2746             default:
2747                 break
2748         }
2749     }
2750     else if (isCrossGenComparisonScenario(scenario)) {
2751         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2752     }
2753     else {
2754         // Skip scenarios
2755         switch (scenario) {
2756             case 'ilrt':
2757                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2758                 // the default scenario build
2759                 if (os != 'Windows_NT') {
2760                     return false
2761                 }
2762                 // Only x64 for now
2763                 if (architecture != 'x64') {
2764                     return false
2765                 }
2766                 // Release only
2767                 if (configuration != 'Release') {
2768                     return false
2769                 }
2770                 break
2771             case 'jitdiff':
2772                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2773                     return false
2774                 }
2775                 if (architecture != 'x64') {
2776                     return false
2777                 }
2778                 if (configuration != 'Checked') {
2779                     return false
2780                 }
2781                 break
2782             case 'longgc':
2783             case 'gcsimulator':
2784                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2785                     return false
2786                 }
2787                 if (architecture != 'x64') {
2788                     return false
2789                 }
2790                 if (configuration != 'Release') {
2791                     return false
2792                 }
2793                 break
2794             case 'gc_reliability_framework':
2795             case 'standalone_gc':
2796                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2797                     return false
2798                 }
2799
2800                 if (architecture != 'x64') {
2801                     return false
2802                 }
2803
2804                 if (configuration != 'Release' && configuration != 'Checked') {
2805                     return false
2806                 }
2807                 break
2808             // We only run Windows and Ubuntu x64 Checked for formatting right now
2809             case 'formatting':
2810                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2811                     return false
2812                 }
2813                 if (architecture != 'x64') {
2814                     return false
2815                 }
2816                 if (configuration != 'Checked') {
2817                     return false
2818                 }
2819                 break
2820             case 'illink':
2821                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2822                     return false
2823                 }
2824                 if (architecture != 'x64' && architecture != 'x86') {
2825                     return false
2826                 }
2827                 break
2828             case 'normal':
2829                 // Nothing skipped
2830                 break
2831             case 'innerloop':
2832                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2833                     return false
2834                 }
2835                 break
2836             case 'corefx_innerloop':
2837                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2838                     return false
2839                 }
2840                 if (architecture != 'x64') {
2841                     return false
2842                 }
2843                 break
2844             default:
2845                 println("Unknown scenario: ${scenario}")
2846                 assert false
2847                 break
2848         }
2849     }
2850
2851     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2852     // runtime behavior, not compile-time behavior, are not interesting.
2853     switch (architecture) {
2854         case 'x86_arm_altjit':
2855         case 'x64_arm64_altjit':
2856             if (isGCStressRelatedTesting(scenario)) {
2857                 return false
2858             }
2859             break
2860         default:
2861             break
2862     }
2863
2864     // The job was not filtered out, so we should generate it!
2865     return true
2866 }
2867
2868 Constants.allScenarios.each { scenario ->
2869     [true, false].each { isPR ->
2870         Constants.architectureList.each { architecture ->
2871             Constants.configurationList.each { configuration ->
2872                 Constants.osList.each { os ->
2873                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2874                     // and reset the os to Windows_NT
2875                     def isBuildOnly = false
2876                     if (os == 'Windows_NT_BuildOnly') {
2877                         isBuildOnly = true
2878                         os = 'Windows_NT'
2879                     }
2880
2881                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2882                         return
2883                     }
2884
2885                     // Calculate names
2886                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2887                     def folderName = getJobFolder(scenario)
2888
2889                     // Create the new job
2890                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2891
2892                     addToViews(newJob, false, isPR, architecture, os, configuration, scenario) // isFlowJob == false
2893
2894                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2895
2896                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2897                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2898                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2899
2900                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2901                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2902                     // artifacts produced.
2903                     if ((architecture == 'armem') && (os == 'Tizen')) {
2904                         // Define the Windows Tests and Corefx build job names
2905                         def lowerConfiguration = configuration.toLowerCase()
2906                         def WindowsTestsName = projectFolder + '/' +
2907                                                Utilities.getFullJobName(project,
2908                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2909                                                                         false)
2910                         def fxBranch = getFxBranch(branch)
2911                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2912                                            Utilities.getFolderName(fxBranch)
2913
2914                         def arm_abi = 'armel'
2915                         def corefx_os = 'tizen'
2916
2917                         // Let's use release CoreFX to test checked CoreCLR,
2918                         // because we do not generate checked CoreFX in CoreFX CI yet.
2919                         def corefx_lowerConfiguration = lowerConfiguration
2920                         if (lowerConfiguration == 'checked') {
2921                             corefx_lowerConfiguration = 'release'
2922                         }
2923
2924                         // Copy the Windows test binaries and the Corefx build binaries
2925                         newJob.with {
2926                             steps {
2927                                 copyArtifacts(WindowsTestsName) {
2928                                     includePatterns('bin/tests/tests.zip')
2929                                     buildSelector {
2930                                         latestSuccessful(true)
2931                                     }
2932                                 }
2933                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2934                                     includePatterns('bin/build.tar.gz')
2935                                     buildSelector {
2936                                         latestSuccessful(true)
2937                                     }
2938                                 }
2939                             } // steps
2940                         } // newJob.with
2941                     }
2942
2943                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2944
2945                     newJob.with {
2946                         steps {
2947                             if (os == 'Windows_NT') {
2948                                 buildCommands.each { buildCommand ->
2949                                     batchFile(buildCommand)
2950                                 }
2951                             }
2952                             else {
2953                                 buildCommands.each { buildCommand ->
2954                                     shell(buildCommand)
2955                                 }
2956                             }
2957                         } // steps
2958                     } // newJob.with
2959
2960                 } // os
2961             } // configuration
2962         } // architecture
2963     } // isPR
2964 } // scenario
2965
2966 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2967 // Returns the newly created job.
2968 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2969 {
2970     def osGroup = getOSGroup(os)
2971     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2972
2973     def jobFolder = getJobFolder(scenario)
2974     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2975         parameters {
2976             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2977         }
2978
2979         steps {
2980             // Set up the copies
2981
2982             // Coreclr build we are trying to test
2983             //
2984             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2985
2986             copyArtifacts(inputCoreCLRBuildName) {
2987                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2988                 buildSelector {
2989                     buildNumber('${CORECLR_BUILD}')
2990                 }
2991             }
2992
2993             if (isCoreFxScenario(scenario)) {
2994
2995                 // Only arm/arm64 supported for corefx testing now.
2996                 assert architecture == 'arm' || architecture == 'arm64'
2997
2998                 // Unzip CoreFx runtime
2999                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3000
3001                 // Unzip CoreFx tests.
3002                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3003
3004                 // Add the script to run the corefx tests
3005                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3006                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
3007                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3008                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3009
3010             } else { // !isCoreFxScenario(scenario)
3011
3012                 // Unzip tests.
3013                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3014
3015                 def buildCommands = ""
3016
3017                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3018                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3019                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3020
3021                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3022                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3023
3024                 // For all jobs
3025                 addEnvVariable("CORE_ROOT", coreRootLocation)
3026                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3027                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3028
3029                 // If we are running a stress mode, we'll set those variables as well
3030                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3031                     def stressValues = null
3032                     if (isJitStressScenario(scenario)) {
3033                         stressValues = Constants.jitStressModeScenarios[scenario]
3034                     }
3035                     else {
3036                         stressValues = Constants.r2rStressScenarios[scenario]
3037                     }
3038
3039                     stressValues.each { key, value ->
3040                         addEnvVariable(key, value)
3041                     }
3042                 }
3043
3044                 if (isR2RScenario(scenario)) {
3045                     // Crossgen the framework assemblies.
3046                     buildCommands += """
3047 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3048 @goto skip_PrecompileAssembly
3049
3050 :PrecompileAssembly
3051 @REM Skip mscorlib since it is already precompiled.
3052 @if /I "%3" == "mscorlib.dll" exit /b 0
3053 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3054
3055 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3056 @if "%errorlevel%" == "-2146230517" (
3057     echo %2 is not a managed assembly.
3058 ) else if "%errorlevel%" == "-2146234344" (
3059     echo %2 is not a managed assembly.
3060 ) else if %errorlevel% neq 0 (
3061     echo Unable to precompile %2
3062 ) else (
3063     echo Precompiled %2
3064 )
3065 @exit /b 0
3066
3067 :skip_PrecompileAssembly
3068 """
3069
3070                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3071                     // crossgen on tests before running them.
3072                     addEnvVariable("RunCrossGen", "true")
3073                 } // isR2RScenario(scenario)
3074
3075                 // Create the smarty command
3076                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3077                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3078                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3079                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3080
3081                 // Exclude tests based on scenario.
3082                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3083                     addArchSpecificExclude(architecture, excludeTag)
3084                 }
3085
3086                 if (isInnerloopTestScenario(scenario)) {
3087                     addExclude("pri1")
3088                 }
3089
3090                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3091                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3092                 addExclude("LONG_RUNNING")
3093
3094                 smartyCommand += "/lstFile Tests.lst"
3095
3096                 def testListArch = [
3097                     'arm64': 'arm64',
3098                     'arm': 'arm'
3099                 ]
3100
3101                 def archLocation = testListArch[architecture]
3102
3103                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3104                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3105                 addCommand("${smartyCommand}")
3106
3107                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3108                 // However, we also need to remove all the variables that were set during this batch file, so we
3109                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3110                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3111                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3112                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3113                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3114                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3115
3116                 // ZIP up the smarty output, no matter what the smarty result.
3117                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3118
3119                 addCommand("echo %errorlevel%")
3120                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3121
3122                 // Use the smarty errorlevel as the script errorlevel.
3123                 addCommand("exit /b %__save_smarty_errorlevel%")
3124
3125                 batchFile(buildCommands)
3126             } // non-corefx testing
3127         } // steps
3128     } // job
3129
3130     if (!isCoreFxScenario(scenario)) {
3131         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3132
3133         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3134         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3135         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3136         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3137     }
3138
3139     return newJob
3140 }
3141
3142 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3143 // E.g., non-Windows tests.
3144 // Returns the newly created job.
3145 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3146 {
3147     def lowerConfiguration = configuration.toLowerCase()
3148
3149     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3150     def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3151     def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3152
3153     def doCoreFxTesting = isCoreFxScenario(scenario)
3154
3155     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3156
3157     def osGroup = getOSGroup(os)
3158     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3159
3160     def testOpts = ''
3161     def useServerGC = false
3162
3163     // Enable Server GC for Ubuntu PR builds
3164     // REVIEW: why? Does this apply to all architectures? Why only PR?
3165     if (os == 'Ubuntu' && isPR) {
3166         testOpts += ' --useServerGC'
3167         useServerGC = true
3168     }
3169
3170     if (isR2RScenario(scenario)) {
3171
3172         testOpts += ' --crossgen --runcrossgentests'
3173
3174         if (scenario == 'r2r_jitstress1') {
3175             testOpts += ' --jitstress=1'
3176         }
3177         else if (scenario == 'r2r_jitstress2') {
3178             testOpts += ' --jitstress=2'
3179         }
3180         else if (scenario == 'r2r_jitstress1_tiered') {
3181             testOpts += ' --jitstress=1'
3182         }
3183         else if (scenario == 'r2r_jitstress2_tiered') {
3184             testOpts += ' --jitstress=2'
3185         }
3186         else if (scenario == 'r2r_jitstressregs1') {
3187             testOpts += ' --jitstressregs=1'
3188         }
3189         else if (scenario == 'r2r_jitstressregs2') {
3190             testOpts += ' --jitstressregs=2'
3191         }
3192         else if (scenario == 'r2r_jitstressregs3') {
3193             testOpts += ' --jitstressregs=3'
3194         }
3195         else if (scenario == 'r2r_jitstressregs4') {
3196             testOpts += ' --jitstressregs=4'
3197         }
3198         else if (scenario == 'r2r_jitstressregs8') {
3199             testOpts += ' --jitstressregs=8'
3200         }
3201         else if (scenario == 'r2r_jitstressregs0x10') {
3202             testOpts += ' --jitstressregs=0x10'
3203         }
3204         else if (scenario == 'r2r_jitstressregs0x80') {
3205             testOpts += ' --jitstressregs=0x80'
3206         }
3207         else if (scenario == 'r2r_jitstressregs0x1000') {
3208             testOpts += ' --jitstressregs=0x1000'
3209         }
3210         else if (scenario == 'r2r_jitminopts') {
3211             testOpts += ' --jitminopts'
3212         }
3213         else if (scenario == 'r2r_jitforcerelocs') {
3214             testOpts += ' --jitforcerelocs'
3215         }
3216         else if (scenario == 'r2r_gcstress15') {
3217             testOpts += ' --gcstresslevel=0xF'
3218         }
3219     }
3220     else if (scenario == 'jitdiff') {
3221         testOpts += ' --jitdisasm --crossgen'
3222     }
3223     else if (scenario == 'illink') {
3224         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3225     }
3226     else if (isLongGc(scenario)) {
3227         // Long GC tests behave very poorly when they are not
3228         // the only test running (many of them allocate until OOM).
3229         testOpts += ' --sequential'
3230
3231         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3232         // for running long GC and GCSimulator tests, respectively. We don't use them
3233         // here because using a playlist file produces much more readable output on the CI machines
3234         // and reduces running time.
3235         //
3236         // The Long GC playlist contains all of the tests that are
3237         // going to be run. The GCSimulator playlist contains all of
3238         // the GC simulator tests.
3239         if (scenario == 'longgc') {
3240             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3241         }
3242         else if (scenario == 'gcsimulator') {
3243             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3244         }
3245     }
3246     else if (isGcReliabilityFramework(scenario)) {
3247         testOpts += ' --build-overlay-only'
3248     }
3249     else if (scenario == 'standalone_gc') {
3250         if (osGroup == 'OSX') {
3251             testOpts += ' --gcname=libclrgc.dylib'
3252         }
3253         else if (osGroup == 'Linux') {
3254             testOpts += ' --gcname=libclrgc.so'
3255         }
3256         else {
3257             println("Unexpected OS group: ${osGroup} for os ${os}")
3258             assert false
3259         }
3260     }
3261
3262     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3263     // will be null in this case.
3264
3265     def jobFolder = getJobFolder(scenario)
3266     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3267         parameters {
3268             if (inputTestsBuildName != null) {
3269                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3270             }
3271             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3272         }
3273
3274         steps {
3275             // Set up the copies
3276
3277             // Coreclr build containing the tests and mscorlib
3278             // pri1 jobs still need to copy windows_nt built tests
3279             if (inputTestsBuildName != null) {
3280                 copyArtifacts(inputTestsBuildName) {
3281                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3282                     buildSelector {
3283                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3284                     }
3285                 }
3286             }
3287
3288             // Coreclr build we are trying to test
3289             //
3290             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3291             //
3292             // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3293             // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3294
3295             if (!isUbuntuArm64Job) {
3296                 copyArtifacts(inputCoreCLRBuildName) {
3297                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3298                     buildSelector {
3299                         buildNumber('${CORECLR_BUILD}')
3300                     }
3301                 }
3302             }
3303
3304             if (isUbuntuArmJob) {
3305                 // Add some useful information to the log file. Ignore return codes.
3306                 shell("uname -a || true")
3307             }
3308
3309             if (isUbuntuArm64Job) {
3310                 // Copy the required artifacts directly, using wget, e.g.:
3311                 //
3312                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3313                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3314                 //
3315                 // parameterized as:
3316                 //
3317                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3318                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3319                 //
3320                 // CoreFX example artifact URLs:
3321                 //
3322                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3323                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3324                 //
3325                 // Note that the source might be in a "jitstress" folder.
3326                 //
3327                 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3328                 //
3329                 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3330
3331                 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3332
3333                 def mungedProjectName = Utilities.getFolderName(project)
3334                 def mungedBranchName = Utilities.getFolderName(branch)
3335
3336                 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3337                 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3338                 if (doCoreFxTesting || doCrossGenComparison) {
3339                     // These depend on unique builds for each scenario
3340                     inputCoreCLRBuildScenario = scenario
3341                 }
3342                 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3343                 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3344
3345                 // Need to add the sub-folder if necessary.
3346                 def inputJobPath = "job/${inputJobName}"
3347                 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3348                 if (folderName != '') {
3349                     inputJobPath = "job/${folderName}/job/${inputJobName}"
3350                 }
3351
3352                 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3353
3354                 if (doCoreFxTesting) {
3355                     shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3356                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3357                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3358                 }
3359                 else {
3360                     shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
3361                     shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
3362                 }
3363             }
3364
3365             if (architecture == 'x86') {
3366                 shell("mkdir ./bin/CoreFxNative")
3367
3368                 def fxBranch = getFxBranch(branch)
3369                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3370
3371                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3372                     includePatterns('bin/build.tar.gz')
3373                     targetDirectory('bin/CoreFxNative')
3374                     buildSelector {
3375                         latestSuccessful(true)
3376                     }
3377                 }
3378
3379                 shell("mkdir ./bin/CoreFxBinDir")
3380                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3381             }
3382
3383             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3384             if (doCoreFxTesting) {
3385                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3386                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3387             }
3388             else if (architecture != 'arm64') {
3389                 // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
3390                 //
3391                 // Unzip the tests first.  Exit with 0
3392                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3393                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3394             }
3395
3396             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3397             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3398             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3399             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3400             // copied correctly.
3401             if (!doCoreFxTesting) {
3402                 if (isUbuntuArmJob) {
3403                     if (architecture == 'arm') {
3404                         shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3405                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3406                     }
3407                     else {
3408                         assert architecture == 'arm64'
3409                         shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
3410
3411                         // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
3412                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3413                     }
3414                 }
3415                 else {
3416                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3417                 }
3418             }
3419
3420             // Execute the tests
3421             def runDocker = isNeedDocker(architecture, os, false)
3422             def dockerPrefix = ""
3423             def dockerCmd = ""
3424             if (runDocker) {
3425                 def dockerImage = getDockerImageName(architecture, os, false)
3426                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3427                 dockerCmd = dockerPrefix + "${dockerImage} "
3428             }
3429
3430             // If we are running a stress mode, we'll set those variables first.
3431             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3432             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3433                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3434                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3435                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3436                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3437                 shell("${envScriptCmds}")
3438                 testOpts += " --test-env=${scriptFileName}"
3439             }
3440
3441             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3442             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3443             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3444             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3445             if (isGCStressRelatedTesting(scenario)) {
3446                 if (architecture == 'x64') {
3447                     shell('./init-tools.sh')
3448                 }
3449             }
3450
3451             if (doCoreFxTesting) {
3452                 shell("""\
3453 \${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3454             }
3455             else {
3456                 def runScript = "${dockerCmd}./tests/runtest.sh"
3457
3458                 // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
3459
3460                 shell("""\
3461 ${runScript} \\
3462     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3463     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3464     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3465     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3466             }
3467
3468             if (isGcReliabilityFramework(scenario)) {
3469                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3470                 if (useServerGC) {
3471                     if (runDocker) {
3472                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3473                     }
3474                     else {
3475                         shell("export COMPlus_gcServer=1")
3476                     }
3477                 }
3478
3479                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3480             }
3481         } // steps
3482     } // job
3483
3484     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3485     if (os in ['Ubuntu']) {
3486         SummaryBuilder summaries = new SummaryBuilder()
3487         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3488         summaries.emit(newJob)
3489     }
3490
3491     if (doCoreFxTesting) {
3492         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3493         if ((os == "Ubuntu") && (architecture == 'arm')) {
3494             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3495             //
3496             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3497             //
3498             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3499             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3500         }
3501         else {
3502             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3503         }
3504     }
3505     else {
3506         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3507     }
3508
3509     return newJob
3510 }
3511
3512 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3513 {
3514     assert isCrossGenComparisonScenario(scenario)
3515
3516     def osGroup = getOSGroup(os)
3517     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3518
3519     def workspaceRelativeResultsDir = "_"
3520     def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3521
3522     def jobFolder = getJobFolder(scenario)
3523     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3524         parameters {
3525             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3526         }
3527
3528         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3529
3530         steps {
3531             copyArtifacts(inputCoreCLRBuildName) {
3532                 includePatterns("${workspaceRelativeArtifactsArchive}")
3533                 buildSelector {
3534                     buildNumber('${CORECLR_BUILD}')
3535                 }
3536             }
3537
3538             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3539
3540             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3541             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3542             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3543             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3544
3545             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3546             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3547
3548             shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3549             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3550             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3551
3552             getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3553                 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3554                 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3555             } // crossArch
3556         } // steps
3557     }  // job
3558
3559     Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**")
3560     getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3561         def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3562         Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**")
3563     } // crossArch
3564
3565     return newJob
3566 }
3567
3568 // Create a test job that will be used by a flow job.
3569 // Returns the newly created job.
3570 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3571 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3572 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3573 {
3574     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3575
3576     def newJob = null
3577     if (windowsArmJob) {
3578         assert inputTestsBuildName == null
3579         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3580     }
3581     else if (isCrossGenComparisonScenario(scenario)) {
3582         assert inputTestsBuildName == null
3583         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3584     }
3585     else {
3586         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3587     }
3588
3589     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3590
3591     if (scenario == 'jitdiff') {
3592         def osGroup = getOSGroup(os)
3593         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3594     }
3595
3596     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3597     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3598
3599     return newJob
3600 }
3601
3602 // Create a flow job to tie together a build job with the given test job.
3603 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3604 // Returns the new flow job.
3605 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3606 {
3607     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3608     // Linux CoreCLR test
3609     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3610     def jobFolder = getJobFolder(scenario)
3611
3612     def newFlowJob = null
3613
3614     if (inputTestsBuildName == null) {
3615         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3616                         buildFlow("""\
3617 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3618
3619 // And then build the test build
3620 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3621 """)
3622         }
3623         JobReport.Report.addReference(inputCoreCLRBuildName)
3624         JobReport.Report.addReference(fullTestJobName)
3625     }
3626     else {
3627         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3628                         buildFlow("""\
3629 // Build the input jobs in parallel
3630 parallel (
3631 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3632 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3633 )
3634
3635 // And then build the test build
3636 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3637                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3638 """)
3639         }
3640         JobReport.Report.addReference(inputCoreCLRBuildName)
3641         JobReport.Report.addReference(inputTestsBuildName)
3642         JobReport.Report.addReference(fullTestJobName)
3643     }
3644
3645     addToViews(newFlowJob, true, isPR, architecture, os, configuration, scenario) // isFlowJob = true
3646
3647     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3648
3649     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3650     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3651
3652     return newFlowJob
3653 }
3654
3655 // Determine if we should generate a flow job for the given parameters.
3656 // Returns true if the job should be generated.
3657 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3658 {
3659     // The various "innerloop" jobs are only available as PR triggered.
3660
3661     if (!isPR) {
3662         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
3663             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
3664             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
3665             return true
3666         }
3667
3668         if (isInnerloopTestScenario(scenario)) {
3669             return false
3670         }
3671
3672         if (scenario == 'corefx_innerloop') {
3673             return false
3674         }
3675     }
3676
3677     // Filter based on OS and architecture.
3678
3679     switch (architecture) {
3680         case 'arm':
3681             if (os != "Ubuntu" && os != "Windows_NT") {
3682                 return false
3683             }
3684             break
3685         case 'arm64':
3686             if (os != "Ubuntu16.04" && os != "Windows_NT") {
3687                 return false
3688             }
3689             break
3690         case 'x86':
3691             if (os != "Ubuntu") {
3692                 return false
3693             }
3694             break
3695         case 'x64':
3696             if (!(os in Constants.crossList)) {
3697                 return false
3698             }
3699             if (os == "Windows_NT") {
3700                 return false
3701             }
3702             break
3703         case 'armem':
3704         case 'x86_arm_altjit':
3705         case 'x64_arm64_altjit':
3706             // No flow jobs
3707             return false
3708         default:
3709             println("Unknown architecture: ${architecture}")
3710             assert false
3711             break
3712     }
3713
3714     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3715
3716     // Filter based on scenario in OS.
3717
3718     if (os == 'Windows_NT') {
3719         assert architecture == 'arm' || architecture == 'arm64'
3720         if (!isArmWindowsScenario(scenario)) {
3721             return false
3722         }
3723         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3724             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3725             // To do that, it doesn't require a flow job.
3726             return false
3727         }
3728     }
3729     else {
3730         // Non-Windows
3731         if (architecture == 'arm') {
3732             if (!(scenario in Constants.validLinuxArmScenarios)) {
3733                 return false
3734             }
3735         }
3736         else if (architecture == 'arm64') {
3737             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3738                 return false
3739             }
3740         }
3741         else if (architecture == 'x86') {
3742             // Linux/x86 only want innerloop and default test
3743             if (!isNormalOrInnerloop) {
3744                 return false
3745             }
3746         }
3747         else if (architecture == 'x64') {
3748             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3749             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3750             // build from a Windows machine.
3751             if (isCoreFxScenario(scenario)) {
3752                 return false
3753             }
3754         }
3755     }
3756
3757     // For CentOS, we only want Checked/Release builds.
3758     if (os == 'CentOS7.1') {
3759         if (configuration != 'Checked' && configuration != 'Release') {
3760             return false
3761         }
3762         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3763             return false
3764         }
3765     }
3766
3767     // For RedHat and Debian, we only do Release builds.
3768     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3769         if (configuration != 'Release') {
3770             return false
3771         }
3772         if (!isNormalOrInnerloop) {
3773             return false
3774         }
3775     }
3776
3777     // Next, filter based on scenario.
3778
3779     if (isJitStressScenario(scenario)) {
3780         if (configuration != 'Checked') {
3781             return false
3782         }
3783     }
3784     else if (isR2RBaselineScenario(scenario)) {
3785         if (configuration != 'Checked' && configuration != 'Release') {
3786             return false
3787         }
3788     }
3789     else if (isR2RStressScenario(scenario)) {
3790         if (configuration != 'Checked') {
3791             return false
3792         }
3793     }
3794     else if (isCrossGenComparisonScenario(scenario)) {
3795         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3796     }
3797     else {
3798         // Skip scenarios
3799         switch (scenario) {
3800             case 'ilrt':
3801             case 'longgc':
3802             case 'gcsimulator':
3803                 // Long GC tests take a long time on non-Release builds
3804                 // ilrt is also Release only
3805                 if (configuration != 'Release') {
3806                     return false
3807                 }
3808                 break
3809
3810             case 'jitdiff':
3811                 if (configuration != 'Checked') {
3812                     return false
3813                 }
3814                 break
3815
3816             case 'gc_reliability_framework':
3817             case 'standalone_gc':
3818                 if (configuration != 'Release' && configuration != 'Checked') {
3819                     return false
3820                 }
3821                 break
3822
3823             case 'formatting':
3824                 return false
3825
3826             case 'illink':
3827                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3828                     return false
3829                 }
3830                 break
3831
3832             case 'normal':
3833                 // Nothing skipped
3834                 break
3835
3836             case 'innerloop':
3837                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3838                     return false
3839                 }
3840                 break
3841
3842             case 'corefx_innerloop':
3843                 // No flow job needed
3844                 return false
3845
3846             default:
3847                 println("Unknown scenario: ${scenario}")
3848                 assert false
3849                 break
3850         }
3851     }
3852
3853     // The job was not filtered out, so we should generate it!
3854     return true
3855 }
3856
3857 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3858 Constants.allScenarios.each { scenario ->
3859     [true, false].each { isPR ->
3860         Constants.architectureList.each { architecture ->
3861             Constants.configurationList.each { configuration ->
3862                 Constants.osList.each { os ->
3863
3864                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3865                         return
3866                     }
3867
3868                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3869                     def doCoreFxTesting = isCoreFxScenario(scenario)
3870                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3871
3872                     // Figure out the job name of the CoreCLR build the test will depend on.
3873
3874                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3875                     def inputCoreCLRBuildIsBuildOnly = false
3876                     if (doCoreFxTesting) {
3877                         // Every CoreFx test depends on its own unique build.
3878                         inputCoreCLRBuildScenario = scenario
3879                         if (windowsArmJob) {
3880                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3881                             inputCoreCLRBuildIsBuildOnly = true
3882                         }
3883                     }
3884                     if (doCrossGenComparison) {
3885                         inputCoreCLRBuildScenario = scenario
3886                     }
3887
3888                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3889                     def inputCoreCLRBuildName = projectFolder + '/' +
3890                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3891
3892                     // Figure out the name of the build job that the test job will depend on.
3893                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3894                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3895                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3896                     // CoreFX build creates the tests.
3897
3898                     def inputTestsBuildName = null
3899
3900                     // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
3901                     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3902
3903                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job) {
3904                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3905
3906                         def inputTestsBuildArch = architecture
3907                         if (architecture == "arm") {
3908                             // Use the x86 test build for arm unix
3909                             inputTestsBuildArch = "x86"
3910                         }
3911
3912                         def inputTestsBuildIsBuildOnly = true
3913
3914                         inputTestsBuildName = projectFolder + '/' +
3915                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3916                     }
3917
3918                     // =============================================================================================
3919                     // Create the test job
3920                     // =============================================================================================
3921
3922                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3923
3924                     // =============================================================================================
3925                     // Create a build flow to join together the build and tests required to run this test.
3926                     // =============================================================================================
3927
3928                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3929                         // Do not create the flow job for RHEL jobs.
3930                         return
3931                     }
3932
3933                     def fullTestJobName = projectFolder + '/' + testJob.name
3934                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3935
3936                 } // os
3937             } // configuration
3938         } // architecture
3939     } // isPR
3940 } // scenario
3941
3942 JobReport.Report.generateJobReport(out)
3943
3944 // Make the call to generate the help job
3945 Utilities.createHelperJob(this, project, branch,
3946     "Welcome to the ${project} Repository",  // This is prepended to the help message
3947     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3948
3949 Utilities.addCROSSCheck(this, project, branch)