netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu'      : 'Linux',
  21                       'Ubuntu16.04' : 'Linux',
  22                       'Ubuntu16.10' : 'Linux',
  23                       'RHEL7.2'     : 'Linux',
  24                       'Debian8.4'   : 'Linux',
  25                       'Fedora24'    : 'Linux',
  26                       'CentOS7.1'   : 'Linux',
  27                       'Tizen'       : 'Linux',
  28                       'OSX10.12'    : 'OSX',
  29                       'Windows_NT'  : 'Windows_NT']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitectures(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return ['x86','x64']
  39
  40         case 'arm64':
  41             return ['x64']
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  51     def static LimitedHardwareBranches = [
  52                'master']
  53
  54     // Innerloop build OS's
  55     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  56     // test execution in the build flow runs.  It generates the exact same build
  57     // as Windows_NT but without running the tests.
  58     def static osList = [
  59                'Ubuntu',
  60                'Debian8.4',
  61                'OSX10.12',
  62                'Windows_NT',
  63                'Windows_NT_BuildOnly',
  64                'CentOS7.1',
  65                'RHEL7.2',
  66                'Ubuntu16.04',
  67                'Ubuntu16.10',
  68                'Fedora24',
  69                'Tizen']
  70
  71     def static crossList = [
  72                'Ubuntu',
  73                'Debian8.4',
  74                'OSX10.12',
  75                'Windows_NT',
  76                'CentOS7.1',
  77                'RHEL7.2']
  78
  79     // This is a set of JIT stress modes combined with the set of variables that
  80     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  81     // the values are the environment variables
  82     def static jitStressModeScenarios = [
  83                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  84                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  85                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  86                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  87                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  88                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  89                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  90                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  91                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  92                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  93                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  94                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  95                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  96                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  97                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  98                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
  99                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 100                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 101                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 102                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 103                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 104                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 105                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 106                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 107                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 108                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 109                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 110                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 111                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 112                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 113                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 114                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 115                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 116                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 117                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 118                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 119                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 120                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 121                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 122                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 123                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 124                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 125                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 126                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 127                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 128                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 129                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 130                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 131                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 132                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 133                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 134                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 135                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 136                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 137                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 138                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 139     ]
 140
 141     // This is a set of ReadyToRun stress scenarios
 142     def static r2rStressScenarios = [
 143                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 144                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 145                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 146                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 147                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 148                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 149                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 150                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 151                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 152                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 153                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 154                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 155                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 156                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 157                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 158                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 159     ]
 160
 161     // This is the basic set of scenarios
 162     def static basicScenarios = [
 163                'innerloop',
 164                'normal',
 165                'ilrt',
 166                'r2r',
 167                'longgc',
 168                'formatting',
 169                'gcsimulator',
 170                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 171                'standalone_gc',
 172                'gc_reliability_framework',
 173                'illink',
 174                'corefx_innerloop',
 175                'crossgen_comparison']
 176
 177     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 178
 179     // Valid PR trigger combinations.
 180     def static prTriggeredValidInnerLoopCombos = [
 181         'Windows_NT': [
 182             'x64': [
 183                 'Checked'
 184             ],
 185             'x86': [
 186                 'Checked',
 187                 'Release'
 188             ],
 189             'arm': [
 190                 'Debug',
 191                 'Checked'
 192             ],
 193             'arm64': [
 194                 'Debug',
 195                 'Checked'
 196             ]
 197         ],
 198         'Windows_NT_BuildOnly': [
 199             'x64': [
 200                 'Checked',
 201                 'Release'
 202             ],
 203             'x86': [
 204                 'Checked',
 205                 'Release'
 206             ],
 207             'arm': [
 208                 'Checked'
 209             ],
 210         ],
 211         'Ubuntu': [
 212             'x64': [
 213                 'Checked'
 214             ],
 215             'arm': [
 216                 'Checked'
 217             ]
 218         ],
 219         'Ubuntu16.04': [
 220             'arm64': [
 221                 'Checked'
 222             ]
 223         ],
 224         'CentOS7.1': [
 225             'x64': [
 226                 'Debug',
 227                 'Checked'
 228             ]
 229         ],
 230         'OSX10.12': [
 231             'x64': [
 232                 'Checked'
 233             ]
 234         ],
 235         'Tizen': [
 236             'armem': [
 237                 'Checked'
 238             ]
 239         ]
 240     ]
 241
 242     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 243     // to Tests.lst file categories to exclude.
 244     //
 245     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 246     // and with the same values, with some commented out, for easier maintenance.
 247     //
 248     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 249     //
 250     def static validArmWindowsScenarios = [
 251                'innerloop':                              [],
 252                'normal':                                 [],
 253                // 'ilrt'
 254                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 255                // 'longgc'
 256                // 'formatting'
 257                // 'gcsimulator'
 258                // 'jitdiff'
 259                // 'standalone_gc'
 260                // 'gc_reliability_framework'
 261                // 'illink'
 262                // 'corefx_innerloop'
 263                // 'crossgen_comparison'
 264                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'r2r_jitstress2_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 273                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 274                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 275                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 276                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 277                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 278                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 279                'r2r_no_tiered_compilation':              ["R2R_FAIL", "R2R_EXCLUDE"],
 280                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 281                'tieredcompilation':                      [],
 282                'no_tiered_compilation':                  [],
 283                'no_tiered_compilation_innerloop':        [],
 284                'forcerelocs':                            [],
 285                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 286                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 287                'jitstress1_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 288                'jitstress2_tiered':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 289                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 290                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 291                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 292                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 293                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 294                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 295                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 296                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 297                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 298                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 300                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 303                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 304                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 305                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 306                // 'jitsse2only'                          // Only relevant to xarch
 307                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 308                // 'jitincompletehwintrinsic'
 309                // 'jitx86hwintrinsicnoavx'
 310                // 'jitx86hwintrinsicnoavx2'
 311                // 'jitx86hwintrinsicnosimd'
 312                // 'jitnox86hwintrinsic'
 313                'corefx_baseline':                        [],    // corefx tests don't use smarty
 314                'corefx_minopts':                         [],    // corefx tests don't use smarty
 315                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 316                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 317                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 318                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 319                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 320                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 321                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 322                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 323                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 324                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 325                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 326                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 327                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 328                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 329                'heapverify1':                            [],
 330                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 331                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 332                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 333                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 334                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 335                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 336
 337                //
 338                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 339                //
 340
 341                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 342                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 343                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 344                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 345                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 346                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 347                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 348                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 349                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 350                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 351                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 352                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 353                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 354                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 355                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 356                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 357                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 358                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 359                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 360     ]
 361
 362     def static validLinuxArmScenarios = [
 363                'innerloop',
 364                'normal',
 365                // 'ilrt'
 366                'r2r',
 367                // 'longgc'
 368                // 'formatting'
 369                // 'gcsimulator'
 370                // 'jitdiff'
 371                // 'standalone_gc'
 372                // 'gc_reliability_framework'
 373                // 'illink'
 374                // 'corefx_innerloop'
 375                'crossgen_comparison',
 376                'r2r_jitstress1',
 377                'r2r_jitstress2',
 378                'r2r_jitstress1_tiered',
 379                'r2r_jitstress2_tiered',
 380                'r2r_jitstressregs1',
 381                'r2r_jitstressregs2',
 382                'r2r_jitstressregs3',
 383                'r2r_jitstressregs4',
 384                'r2r_jitstressregs8',
 385                'r2r_jitstressregs0x10',
 386                'r2r_jitstressregs0x80',
 387                'r2r_jitstressregs0x1000',
 388                'r2r_jitminopts',
 389                'r2r_jitforcerelocs',
 390                'r2r_gcstress15',
 391                'r2r_no_tiered_compilation',
 392                'minopts',
 393                'tieredcompilation',
 394                'no_tiered_compilation',
 395                'no_tiered_compilation_innerloop',
 396                'forcerelocs',
 397                'jitstress1',
 398                'jitstress2',
 399                'jitstress1_tiered',
 400                'jitstress2_tiered',
 401                'jitstressregs1',
 402                'jitstressregs2',
 403                'jitstressregs3',
 404                'jitstressregs4',
 405                'jitstressregs8',
 406                'jitstressregs0x10',
 407                'jitstressregs0x80',
 408                'jitstressregs0x1000',
 409                'jitstress2_jitstressregs1',
 410                'jitstress2_jitstressregs2',
 411                'jitstress2_jitstressregs3',
 412                'jitstress2_jitstressregs4',
 413                'jitstress2_jitstressregs8',
 414                'jitstress2_jitstressregs0x10',
 415                'jitstress2_jitstressregs0x80',
 416                'jitstress2_jitstressregs0x1000',
 417                'tailcallstress',
 418                // 'jitsse2only'                          // Only relevant to xarch
 419                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 420                // 'jitincompletehwintrinsic'
 421                // 'jitx86hwintrinsicnoavx'
 422                // 'jitx86hwintrinsicnoavx2'
 423                // 'jitx86hwintrinsicnosimd'
 424                // 'jitnox86hwintrinsic'
 425                'corefx_baseline',
 426                'corefx_minopts',
 427                'corefx_tieredcompilation',
 428                'corefx_jitstress1',
 429                'corefx_jitstress2',
 430                'corefx_jitstressregs1',
 431                'corefx_jitstressregs2',
 432                'corefx_jitstressregs3',
 433                'corefx_jitstressregs4',
 434                'corefx_jitstressregs8',
 435                'corefx_jitstressregs0x10',
 436                'corefx_jitstressregs0x80',
 437                'corefx_jitstressregs0x1000',
 438                'gcstress0x3',
 439                'gcstress0xc',
 440                'zapdisable',
 441                'heapverify1',
 442                'gcstress0xc_zapdisable',
 443                'gcstress0xc_zapdisable_jitstress2',
 444                'gcstress0xc_zapdisable_heapverify1',
 445                'gcstress0xc_jitstress1',
 446                'gcstress0xc_jitstress2',
 447                'gcstress0xc_minopts_heapverify1'
 448     ]
 449
 450     def static validLinuxArm64Scenarios = [
 451                'innerloop',
 452                'normal',
 453                // 'ilrt'
 454                'r2r',
 455                // 'longgc'
 456                // 'formatting'
 457                // 'gcsimulator'
 458                // 'jitdiff'
 459                // 'standalone_gc'
 460                // 'gc_reliability_framework'
 461                // 'illink'
 462                // 'corefx_innerloop'
 463                // 'crossgen_comparison'
 464                'r2r_jitstress1',
 465                'r2r_jitstress2',
 466                'r2r_jitstress1_tiered',
 467                'r2r_jitstress2_tiered',
 468                'r2r_jitstressregs1',
 469                'r2r_jitstressregs2',
 470                'r2r_jitstressregs3',
 471                'r2r_jitstressregs4',
 472                'r2r_jitstressregs8',
 473                'r2r_jitstressregs0x10',
 474                'r2r_jitstressregs0x80',
 475                'r2r_jitstressregs0x1000',
 476                'r2r_jitminopts',
 477                'r2r_jitforcerelocs',
 478                'r2r_gcstress15',
 479                'r2r_no_tiered_compilation',
 480                'minopts',
 481                'tieredcompilation',
 482                'no_tiered_compilation',
 483                'no_tiered_compilation_innerloop',
 484                'forcerelocs',
 485                'jitstress1',
 486                'jitstress2',
 487                'jitstress1_tiered',
 488                'jitstress2_tiered',
 489                'jitstressregs1',
 490                'jitstressregs2',
 491                'jitstressregs3',
 492                'jitstressregs4',
 493                'jitstressregs8',
 494                'jitstressregs0x10',
 495                'jitstressregs0x80',
 496                'jitstressregs0x1000',
 497                'jitstress2_jitstressregs1',
 498                'jitstress2_jitstressregs2',
 499                'jitstress2_jitstressregs3',
 500                'jitstress2_jitstressregs4',
 501                'jitstress2_jitstressregs8',
 502                'jitstress2_jitstressregs0x10',
 503                'jitstress2_jitstressregs0x80',
 504                'jitstress2_jitstressregs0x1000',
 505                'tailcallstress',
 506                // 'jitsse2only'                         // Only relevant to xarch
 507                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 508                // 'jitincompletehwintrinsic'
 509                // 'jitx86hwintrinsicnoavx'
 510                // 'jitx86hwintrinsicnoavx2'
 511                // 'jitx86hwintrinsicnosimd'
 512                // 'jitnox86hwintrinsic'
 513                'corefx_baseline',
 514                'corefx_minopts',
 515                'corefx_tieredcompilation',
 516                'corefx_jitstress1',
 517                'corefx_jitstress2',
 518                'corefx_jitstressregs1',
 519                'corefx_jitstressregs2',
 520                'corefx_jitstressregs3',
 521                'corefx_jitstressregs4',
 522                'corefx_jitstressregs8',
 523                'corefx_jitstressregs0x10',
 524                'corefx_jitstressregs0x80',
 525                'corefx_jitstressregs0x1000',
 526                'gcstress0x3',
 527                'gcstress0xc',
 528                'zapdisable',
 529                'heapverify1',
 530                'gcstress0xc_zapdisable',
 531                'gcstress0xc_zapdisable_jitstress2',
 532                'gcstress0xc_zapdisable_heapverify1',
 533                'gcstress0xc_jitstress1',
 534                'gcstress0xc_jitstress2',
 535                'gcstress0xc_minopts_heapverify1'
 536     ]
 537
 538     def static configurationList = ['Debug', 'Checked', 'Release']
 539
 540     // This is the set of architectures
 541     // Some of these are pseudo-architectures:
 542     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 543     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 544     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 545     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 546
 547     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 548     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 549 }
 550
 551 // **************************************************************
 552 // Create some specific views
 553 //
 554 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 555 // views based on a single regular expression. These views will be generated by adding a
 556 // specific set of jobs to them.
 557 //
 558 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 559 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 560 // those are required, don't add them (which simplifies the view pages, as well).
 561 // **************************************************************
 562
 563 class Views {
 564     def static MergeJobView = null
 565     def static PeriodicJobView = null
 566     def static ArchitectureViews = [:]
 567     def static OSViews = [:]
 568 }
 569
 570 // MergeJobView: include all jobs that execute when a PR change is merged.
 571 Views.MergeJobView = listView('Merge') {
 572     recurse()
 573     columns {
 574         status()
 575         weather()
 576         name()
 577         lastSuccess()
 578         lastFailure()
 579         lastDuration()
 580         buildButton()
 581     }
 582 }
 583
 584 // PeriodicJobView: include all jobs that execute on a schedule
 585 Views.PeriodicJobView = listView('Periodic') {
 586     recurse()
 587     columns {
 588         status()
 589         weather()
 590         name()
 591         lastSuccess()
 592         lastFailure()
 593         lastDuration()
 594         buildButton()
 595     }
 596 }
 597
 598 // Create a view for non-PR jobs for each architecture.
 599 Constants.architectureList.each { architecture ->
 600     Views.ArchitectureViews[architecture] = listView(architecture) {
 601         recurse()
 602         columns {
 603             status()
 604             weather()
 605             name()
 606             lastSuccess()
 607             lastFailure()
 608             lastDuration()
 609             buildButton()
 610         }
 611     }
 612 }
 613
 614 // Create a view for non-PR jobs for each OS.
 615 Constants.osList.each { os ->
 616     // Don't create one for the special 'Windows_NT_BuildOnly'
 617     if (os == 'Windows_NT_BuildOnly') {
 618         return
 619     }
 620     Views.OSViews[os] = listView(os) {
 621         recurse()
 622         columns {
 623             status()
 624             weather()
 625             name()
 626             lastSuccess()
 627             lastFailure()
 628             lastDuration()
 629             buildButton()
 630         }
 631     }
 632 }
 633
 634 def static addToMergeView(def job) {
 635     Views.MergeJobView.with {
 636         jobs {
 637             name(job.name)
 638         }
 639     }
 640 }
 641
 642 def static addToPeriodicView(def job) {
 643     Views.PeriodicJobView.with {
 644         jobs {
 645             name(job.name)
 646         }
 647     }
 648 }
 649
 650 def static addToViews(def job, def isPR, def architecture, def os) {
 651     if (isPR) {
 652         // No views want PR jobs currently.
 653         return
 654     }
 655
 656     // Add to architecture view.
 657     Views.ArchitectureViews[architecture].with {
 658         jobs {
 659             name(job.name)
 660         }
 661     }
 662
 663     // Add to OS view.
 664     Views.OSViews[os].with {
 665         jobs {
 666             name(job.name)
 667         }
 668     }
 669 }
 670
 671 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 672     addToPeriodicView(job)
 673     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 674 }
 675
 676 def static addGithubPushTriggerHelper(def job) {
 677     addToMergeView(job)
 678     Utilities.addGithubPushTrigger(job)
 679 }
 680
 681
 682 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 683     assert os instanceof String
 684     assert architecture instanceof String
 685
 686     def armArches = ['arm', 'armem', 'arm64']
 687
 688     if (!(architecture in armArches)) {
 689         assert options == null
 690         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 691
 692         return
 693     }
 694
 695     // This is an arm(64) job.
 696     //
 697     // There are several options.
 698     //
 699     // Windows_NT
 700     //
 701     // Arm32 (Build) -> latest-arm64
 702     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 703     // Arm32 (Test)  -> arm64-windows_nt
 704     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 705     //
 706     // Arm64 (Build) -> latest-arm64
 707     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 708     // Arm64 (Test)  -> arm64-windows_nt
 709     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 710     //
 711     // Ubuntu
 712     //
 713     // Arm32 emulator (Build, Test) -> arm-cross-latest
 714     //       |-> os == "Tizen" && (architecture == "armem")
 715     //
 716     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 717     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 718     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 719     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 720     // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
 721     //       |-> os == "Ubuntu" && (architecture == "arm")
 722     //
 723     // Arm64 (Build) -> arm64-cross-latest
 724     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
 725     // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
 726     //       |-> os != "Windows_NT" && architecture == "arm64"
 727     //
 728     // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
 729     // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
 730     //
 731     // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
 732     // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 733
 734     // This has to be a arm arch
 735     assert architecture in armArches
 736     if (os == "Windows_NT") {
 737         // arm32/arm64 Windows jobs share the same machines for now
 738         def isBuild = options['use_arm64_build_machine'] == true
 739
 740         if (isBuild == true) {
 741             job.with {
 742                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 743             }
 744         } else {
 745             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 746         }
 747     } else {
 748         assert os != 'Windows_NT'
 749
 750         if (architecture == 'armem') {
 751             // arm emulator (Tizen). Build and test on same machine,
 752             // using Docker.
 753             assert os == 'Tizen'
 754             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 755         }
 756         else {
 757             // arm/arm64 Ubuntu on hardware.
 758             assert architecture == 'arm' || architecture == 'arm64'
 759             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 760             def isBuild = (options != null) && (options['is_build_job'] == true)
 761             if (isFlow || isBuild) {
 762                 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 763                 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
 764                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 765             } else {
 766                 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
 767                 if (architecture == 'arm64') {
 768                     assert os == 'Ubuntu16.04'
 769                     job.with {
 770                         label('Ubuntu.1604.Arm64.Open')
 771                     }
 772                 }
 773                 else {
 774                     assert os == 'Ubuntu'
 775                     job.with {
 776                         label('ubuntu.1404.arm32.open')
 777                     }
 778                 }
 779             }
 780         }
 781     }
 782 }
 783
 784 // setJobMachineAffinity: compute the machine affinity options for a job,
 785 // then set the job with those affinity options.
 786 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 787 {
 788     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 789            (!isBuildJob && isTestJob  && !isFlowJob) ||
 790            (!isBuildJob && !isTestJob && isFlowJob)
 791
 792     def affinityOptions = null
 793     def affinityArchitecture = architecture
 794
 795     if (os == "Windows_NT") {
 796         if (architecture in Constants.armWindowsCrossArchitectureList) {
 797             if (isBuildJob) {
 798                 affinityOptions = [ "use_arm64_build_machine" : true ]
 799             } else if (isTestJob) {
 800                 affinityOptions = [ "use_arm64_build_machine" : false ]
 801             } else if (isFlowJob) {
 802                 // For the flow jobs set the machine affinity as x64
 803                 affinityArchitecture = 'x64'
 804             }
 805         }
 806     }
 807     else {
 808         if ((architecture == 'arm64') || (architecture == 'arm')) {
 809             if (isBuildJob) {
 810                 affinityOptions = ['is_build_job': true]
 811             } else if (isFlowJob) {
 812                 affinityOptions = ['is_flow_job': true]
 813             }
 814         }
 815     }
 816
 817     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 818 }
 819
 820 def static isGCStressRelatedTesting(def scenario) {
 821     // The 'r2r_gcstress15' scenario is a basic scenario.
 822     // Detect it and make it a GCStress related.
 823     if (scenario == 'r2r_gcstress15')
 824     {
 825         return true;
 826     }
 827
 828     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 829     def scenarioName = scenario.toLowerCase()
 830     def isGCStressTesting = false
 831     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 832         if (k in gcStressTestEnvVars) {
 833             isGCStressTesting = true;
 834         }
 835     }
 836     return isGCStressTesting
 837 }
 838
 839 def static isCoreFxScenario(def scenario) {
 840     def corefx_prefix = 'corefx_'
 841     if (scenario.length() < corefx_prefix.length()) {
 842         return false
 843     }
 844     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 845 }
 846
 847 def static isR2RBaselineScenario(def scenario) {
 848     return (scenario == 'r2r')
 849 }
 850
 851 def static isR2RStressScenario(def scenario) {
 852     return Constants.r2rStressScenarios.containsKey(scenario)
 853 }
 854
 855 def static isR2RScenario(def scenario) {
 856     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 857 }
 858
 859 def static isJitStressScenario(def scenario) {
 860     return Constants.jitStressModeScenarios.containsKey(scenario)
 861 }
 862
 863 def static isLongGc(def scenario) {
 864     return (scenario == 'longgc' || scenario == 'gcsimulator')
 865 }
 866
 867 def static isJitDiff(def scenario) {
 868     return (scenario == 'jitdiff')
 869 }
 870
 871 def static isGcReliabilityFramework(def scenario) {
 872     return (scenario == 'gc_reliability_framework')
 873 }
 874
 875 def static isArmWindowsScenario(def scenario) {
 876     return Constants.validArmWindowsScenarios.containsKey(scenario)
 877 }
 878
 879 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 880     if (isBuildOnly == true) {
 881         os = 'Windows_NT_BuildOnly'
 882     }
 883
 884     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 885     if (validOsPrTriggerArchConfigs != null) {
 886         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 887         if (validOsPrTriggerConfigs != null) {
 888             if (configuration in validOsPrTriggerConfigs) {
 889                 return true
 890             }
 891         }
 892     }
 893
 894     return false
 895 }
 896
 897 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 898 // scheduled with a default PR trigger despite the correlation being true at the moment.
 899 def static isInnerloopTestScenario(def scenario) {
 900     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 901 }
 902
 903 def static isCrossGenComparisonScenario(def scenario) {
 904     return (scenario == 'crossgen_comparison')
 905 }
 906
 907 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 908     assert isCrossGenComparisonScenario(scenario)
 909     return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
 910 }
 911
 912 def static getFxBranch(def branch) {
 913     def fxBranch = branch
 914     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 915     // branch even though CoreFX doesn't have such a branch.
 916     if (branch == 'dev/unix_test_workflow') {
 917         fxBranch = 'master'
 918     }
 919     return fxBranch
 920 }
 921
 922 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 923     // 2 hours (120 minutes) is the default timeout
 924     def timeout = 120
 925
 926     if (!isInnerloopTestScenario(scenario)) {
 927         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 928         timeout = 240
 929     }
 930
 931     if (!isBuildOnly) {
 932         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 933         if (isGCStressRelatedTesting(scenario)) {
 934             timeout = 4320
 935         }
 936         else if (isCoreFxScenario(scenario)) {
 937             timeout = 360
 938         }
 939         else if (isJitStressScenario(scenario)) {
 940             timeout = 300
 941         }
 942         else if (isR2RBaselineScenario(scenario)) {
 943             timeout = 240
 944         }
 945         else if (isLongGc(scenario)) {
 946             timeout = 1440
 947         }
 948         else if (isJitDiff(scenario)) {
 949             timeout = 240
 950         }
 951         else if (isGcReliabilityFramework(scenario)) {
 952             timeout = 1440
 953         }
 954         else if (architecture == 'armem' || architecture == 'arm64') {
 955             timeout = 240
 956         }
 957
 958         if (architecture == 'arm') {
 959             // ARM32 machines are particularly slow.
 960             timeout += 120
 961         }
 962     }
 963
 964     if (configuration == 'Debug') {
 965         // Debug runs can be very slow. Add an hour.
 966         timeout += 60
 967     }
 968
 969     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 970         // AltJit runs compile all methods twice.
 971         timeout *= 2
 972     }
 973
 974     // If we've changed the timeout from the default, set it in the job.
 975
 976     if (timeout != 120) {
 977         Utilities.setJobTimeout(newJob, timeout)
 978     }
 979 }
 980
 981 def static getJobFolder(def scenario) {
 982     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 983         return 'jitstress'
 984     }
 985     if (scenario == 'illink') {
 986         return 'illink'
 987     }
 988     return ''
 989 }
 990
 991 def static getStressModeDisplayName(def scenario) {
 992     def displayStr = ''
 993     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 994         def prefixLength = 'COMPlus_'.length()
 995         if (k.length() >= prefixLength) {
 996             def modeName = k.substring(prefixLength, k.length())
 997             if (displayStr != '') {
 998                 // Separate multiple variables with a space.
 999                 displayStr += ' '
1000             }
1001             displayStr += modeName + '=' + v
1002         }
1003     }
1004
1005     if (isCoreFxScenario(scenario)) {
1006         displayStr = ('CoreFx ' + displayStr).trim()
1007     }
1008
1009     return displayStr
1010 }
1011
1012 def static getR2RDisplayName(def scenario) {
1013     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1014     def displayStr = scenario
1015     def prefixLength = 'r2r_'.length()
1016     if (displayStr.length() >= prefixLength) {
1017         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1018     } else if (scenario == 'r2r') {
1019         displayStr = "R2R"
1020     }
1021     return displayStr
1022 }
1023
1024 def static getScenarioDisplayString(def scenario) {
1025     switch (scenario) {
1026         case 'innerloop':
1027             return "Innerloop Build and Test"
1028
1029         case 'no_tiered_compilation_innerloop':
1030             def displayStr = getStressModeDisplayName(scenario)
1031             return "Innerloop Build and Test (Jit - ${displayStr})"
1032
1033         case 'corefx_innerloop':
1034             return "CoreFX Tests"
1035
1036         case 'normal':
1037             return "Build and Test"
1038
1039         case 'jitdiff':
1040             return "Jit Diff Build and Test"
1041
1042         case 'ilrt':
1043             return "IL RoundTrip Build and Test"
1044
1045         case 'longgc':
1046             return "Long-Running GC Build & Test"
1047
1048         case 'gcsimulator':
1049             return "GC Simulator"
1050
1051         case 'standalone_gc':
1052             return "Standalone GC"
1053
1054         case 'gc_reliability_framework':
1055             return "GC Reliability Framework"
1056
1057         case 'illink':
1058             return "via ILLink"
1059
1060         default:
1061             if (isJitStressScenario(scenario)) {
1062                 def displayStr = getStressModeDisplayName(scenario)
1063                 return "Build and Test (Jit - ${displayStr})"
1064             }
1065             else if (isR2RScenario(scenario)) {
1066                 def displayStr = getR2RDisplayName(scenario)
1067                 return "${displayStr} Build and Test"
1068             }
1069             else {
1070                 return "${scenario}"
1071             }
1072             break
1073     }
1074
1075     println("Unknown scenario: ${scenario}");
1076     assert false
1077 }
1078
1079 //
1080 // Functions to create an environment script.
1081 //      envScriptCreate -- initialize the script (call first)
1082 //      envScriptFinalize -- finalize the script (call last)
1083 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1084 //      envScriptAppendExistingScript -- append an existing script to the generated script
1085 //
1086 // Each script returns a string of commands. Concatenate all the strings together before
1087 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1088 //
1089
1090 // Initialize the environment setting script.
1091 def static envScriptCreate(def os, def stepScriptLocation) {
1092     def stepScript = ''
1093     if (os == 'Windows_NT') {
1094         stepScript += "echo Creating TestEnv script\r\n"
1095         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1096
1097         // Create at least an empty script.
1098         stepScript += "echo. > ${stepScriptLocation}\r\n"
1099     }
1100     else {
1101         stepScript += "echo Creating environment setting script\n"
1102         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1103     }
1104
1105     return stepScript
1106 }
1107
1108 // Generates the string for setting stress mode variables.
1109 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1110     def stepScript = ''
1111     if (os == 'Windows_NT') {
1112         stressModeVars.each{ k, v ->
1113             // Write out what we are writing to the script file
1114             stepScript += "echo Setting ${k}=${v}\r\n"
1115             // Write out the set itself to the script file`
1116             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1117         }
1118     }
1119     else {
1120         stressModeVars.each{ k, v ->
1121             // Write out what we are writing to the script file
1122             stepScript += "echo Setting ${k}=${v}\n"
1123             // Write out the set itself to the script file`
1124             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1125         }
1126     }
1127
1128     return stepScript
1129 }
1130
1131 // Append an existing script to an environment script.
1132 // Returns string of commands to do this.
1133 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1134     assert (os == 'Windows_NT')
1135     def stepScript = ''
1136
1137     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1138     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1139
1140     return stepScript
1141 }
1142
1143 // Finalize an environment setting script.
1144 // Returns string of commands to do this.
1145 def static envScriptFinalize(def os, def stepScriptLocation) {
1146     def stepScript = ''
1147
1148     if (os == 'Windows_NT') {
1149         // Display the resulting script. This is useful when looking at the output log file.
1150         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1151         stepScript += "type ${stepScriptLocation}\r\n"
1152     }
1153     else {
1154         stepScript += "chmod +x ${stepScriptLocation}\n"
1155     }
1156
1157     return stepScript
1158 }
1159
1160 def static isNeedDocker(def architecture, def os, def isBuild) {
1161     if (isBuild) {
1162         if (architecture == 'x86' && os == 'Ubuntu') {
1163             return true
1164         }
1165         else if (architecture == 'armem') {
1166             return true
1167         }
1168         else if (architecture == 'arm') {
1169             if (os == 'Ubuntu') {
1170                 return true
1171             }
1172         }
1173         else if (architecture == 'arm64') {
1174             if (os == 'Ubuntu16.04') {
1175                 return true
1176             }
1177         }
1178     }
1179     else {
1180         if (architecture == 'x86' && os == 'Ubuntu') {
1181             return true
1182         }
1183     }
1184     return false
1185 }
1186
1187 def static getDockerImageName(def architecture, def os, def isBuild) {
1188     // We must change some docker private images to official later
1189     if (isBuild) {
1190         if (architecture == 'x86' && os == 'Ubuntu') {
1191             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1192         }
1193         else if (architecture == 'armem') {
1194             if (os == 'Tizen') {
1195                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1196             }
1197         }
1198         else if (architecture == 'arm') {
1199             if (os == 'Ubuntu') {
1200                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1201             }
1202         }
1203         else if (architecture == 'arm64') {
1204             if (os == 'Ubuntu16.04') {
1205                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1206             }
1207         }
1208     }
1209     else {
1210         if (architecture == 'x86' && os == 'Ubuntu') {
1211             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1212         }
1213     }
1214     println("Unknown architecture to use docker: ${architecture} ${os}");
1215     assert false
1216 }
1217
1218
1219 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1220 // and only allowing using this hardware in some specific branches.
1221 def static jobRequiresLimitedHardware(def architecture, def os) {
1222     if (architecture == 'arm') {
1223         // arm Windows and Linux hardware is limited.
1224         return true
1225     }
1226     else if (architecture == 'arm64') {
1227         if (os == 'Windows_NT') {
1228             // arm64 Windows hardware is limited.
1229             return true
1230         }
1231         else {
1232             // arm64 Linux hardware is fast enough to allow more frequent jobs
1233             return false
1234         }
1235     }
1236     else {
1237         return false
1238     }
1239 }
1240
1241 // Calculates the name of the build job based on some typical parameters.
1242 //
1243 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1244     // If the architecture is x64, do not add that info into the build name.
1245     // Need to change around some systems and other builds to pick up the right builds
1246     // to do that.
1247
1248     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1249     if (isBuildOnly) {
1250         suffix += '_bld'
1251     }
1252     def baseName = ''
1253     switch (architecture) {
1254         case 'x64':
1255             if (scenario == 'normal') {
1256                 // For now we leave x64 off of the name for compatibility with other jobs
1257                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1258             }
1259             else if (scenario == 'formatting') {
1260                 // we don't care about the configuration for the formatting job. It runs all configs
1261                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1262             }
1263             else {
1264                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1265             }
1266             break
1267         case 'armem':
1268             // These are cross builds
1269             assert os == 'Tizen'
1270             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1271             break
1272         case 'arm':
1273         case 'arm64':
1274             // These are cross builds
1275             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1276             break
1277         case 'x86':
1278         case 'x86_arm_altjit':
1279         case 'x64_arm64_altjit':
1280             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1281             break
1282         default:
1283             println("Unknown architecture: ${architecture}");
1284             assert false
1285             break
1286     }
1287
1288     return baseName + suffix
1289 }
1290
1291 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1292     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1293
1294     // Limited hardware is restricted for non-PR triggers to certain branches.
1295     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1296         return
1297     }
1298
1299     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1300     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1301     if (architecture == 'x86' && os == 'Ubuntu') {
1302         return
1303     }
1304
1305     // Check scenario.
1306     switch (scenario) {
1307         case 'innerloop':
1308         case 'no_tiered_compilation_innerloop':
1309             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1310             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1311             // We should never get here (in the "innerloop cases) for non-PR jobs, except for this TEMPORARY exception.
1312             // Only trigger the flow job, not the build job.
1313             assert (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked'))
1314             if (isFlowJob) {
1315                 addGithubPushTriggerHelper(job)
1316             }
1317             break
1318
1319         case 'crossgen_comparison':
1320             break
1321         case 'normal':
1322             switch (architecture) {
1323                 case 'x64':
1324                 case 'x86':
1325                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1326                         addPeriodicTriggerHelper(job, '@daily')
1327                     }
1328                     else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1329                         addGithubPushTriggerHelper(job)
1330                     }
1331                     break
1332                 case 'arm64':
1333                     if (os == 'Windows_NT') {
1334                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1335                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1336                             // Do the builds daily.
1337                             addPeriodicTriggerHelper(job, '@daily')
1338                         }
1339                     }
1340                     else {
1341                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1342                         if (isFlowJob) {
1343                             addPeriodicTriggerHelper(job, '@daily')
1344                         }
1345                     }
1346                     break
1347                 case 'arm':
1348                     if (os == 'Windows_NT') {
1349                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1350                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1351                             // Do the builds daily.
1352                             addPeriodicTriggerHelper(job, '@daily')
1353                         }
1354                     }
1355                     else {
1356                         assert os == 'Ubuntu'
1357                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1358                         if (isFlowJob) {
1359                             // Currently no push triggers, with limited arm Linux hardware.
1360                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1361                             addPeriodicTriggerHelper(job, '@daily')
1362                         }
1363                     }
1364                     break
1365                 case 'armem':
1366                     addGithubPushTriggerHelper(job)
1367                     break
1368                 case 'x86_arm_altjit':
1369                 case 'x64_arm64_altjit':
1370                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1371                     if (configuration == 'Checked') {
1372                         addGithubPushTriggerHelper(job)
1373                     }
1374                     break
1375                 default:
1376                     println("Unknown architecture: ${architecture}");
1377                     assert false
1378                     break
1379             }
1380             break
1381         case 'r2r':
1382             assert !(os in bidailyCrossList)
1383             // r2r gets a push trigger for checked/release
1384             if (configuration == 'Checked' || configuration == 'Release') {
1385                 if (architecture == 'x64' && os != 'OSX10.12') {
1386                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1387                     if (isFlowJob || os == 'Windows_NT') {
1388                         addGithubPushTriggerHelper(job)
1389                     }
1390                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1391                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1392                     if (isFlowJob) {
1393                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1394                     }
1395                 }
1396                 // For x86, only add per-commit jobs for Windows
1397                 else if (architecture == 'x86') {
1398                     if (os == 'Windows_NT') {
1399                         addGithubPushTriggerHelper(job)
1400                     }
1401                 }
1402                 // arm r2r jobs should only run weekly.
1403                 else if (architecture == 'arm') {
1404                     if (isFlowJob) {
1405                         addPeriodicTriggerHelper(job, '@weekly')
1406                     }
1407                 }
1408                 // arm64 r2r jobs should only run weekly.
1409                 else if (architecture == 'arm64') {
1410                     if (isFlowJob) {
1411                         addPeriodicTriggerHelper(job, '@weekly')
1412                     }
1413                 }
1414             }
1415             break
1416         case 'r2r_jitstress1':
1417         case 'r2r_jitstress2':
1418         case 'r2r_jitstress1_tiered':
1419         case 'r2r_jitstress2_tiered':
1420         case 'r2r_jitstressregs1':
1421         case 'r2r_jitstressregs2':
1422         case 'r2r_jitstressregs3':
1423         case 'r2r_jitstressregs4':
1424         case 'r2r_jitstressregs8':
1425         case 'r2r_jitstressregs0x10':
1426         case 'r2r_jitstressregs0x80':
1427         case 'r2r_jitstressregs0x1000':
1428         case 'r2r_jitminopts':
1429         case 'r2r_jitforcerelocs':
1430         case 'r2r_gcstress15':
1431         case 'r2r_no_tiered_compilation':
1432             assert !(os in bidailyCrossList)
1433
1434             // GCStress=C is currently not supported on OS X
1435             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1436                 break
1437             }
1438
1439             if (configuration == 'Checked' || configuration == 'Release') {
1440                 if (architecture == 'x64') {
1441                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1442                     if (isFlowJob || os == 'Windows_NT') {
1443                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1444                     }
1445                 }
1446                 // For x86, only add periodic jobs for Windows
1447                 else if (architecture == 'x86') {
1448                     if (os == 'Windows_NT') {
1449                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1450                     }
1451                 }
1452                 else if (architecture == 'arm') {
1453                     if (isFlowJob) {
1454                         addPeriodicTriggerHelper(job, '@weekly')
1455                     }
1456                 }
1457                 else if (architecture == 'arm64') {
1458                     if (isFlowJob) {
1459                         addPeriodicTriggerHelper(job, '@weekly')
1460                     }
1461                 }
1462             }
1463             break
1464         case 'longgc':
1465             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1466             assert configuration == 'Release'
1467             assert architecture == 'x64'
1468             addPeriodicTriggerHelper(job, '@daily')
1469             // TODO: Add once external email sending is available again
1470             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1471             break
1472         case 'gcsimulator':
1473             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1474             assert configuration == 'Release'
1475             assert architecture == 'x64'
1476             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1477             // TODO: Add once external email sending is available again
1478             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1479             break
1480         case 'standalone_gc':
1481             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1482             assert (configuration == 'Release' || configuration == 'Checked')
1483             // TODO: Add once external email sending is available again
1484             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1485             addPeriodicTriggerHelper(job, '@daily')
1486             break
1487         case 'gc_reliability_framework':
1488             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1489             assert (configuration == 'Release' || configuration == 'Checked')
1490             // Only triggered by phrase.
1491             break
1492         case 'ilrt':
1493             assert !(os in bidailyCrossList)
1494             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1495             if (architecture == 'x64' && configuration == 'Release') {
1496                 if (isFlowJob || os == 'Windows_NT') {
1497                     addPeriodicTriggerHelper(job, '@daily')
1498                 }
1499             }
1500             break
1501         case 'jitdiff':
1502             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1503             assert configuration == 'Checked'
1504             assert (architecture == 'x64' || architecture == 'x86')
1505             addGithubPushTriggerHelper(job)
1506             break
1507         case 'formatting':
1508             assert (os == 'Windows_NT' || os == "Ubuntu")
1509             assert architecture == 'x64'
1510             addGithubPushTriggerHelper(job)
1511             break
1512         case 'jitstressregs1':
1513         case 'jitstressregs2':
1514         case 'jitstressregs3':
1515         case 'jitstressregs4':
1516         case 'jitstressregs8':
1517         case 'jitstressregs0x10':
1518         case 'jitstressregs0x80':
1519         case 'jitstressregs0x1000':
1520         case 'minopts':
1521         case 'tieredcompilation':
1522         case 'no_tiered_compilation':
1523         case 'forcerelocs':
1524         case 'jitstress1':
1525         case 'jitstress2':
1526         case 'jitstress1_tiered':
1527         case 'jitstress2_tiered':
1528         case 'jitstress2_jitstressregs1':
1529         case 'jitstress2_jitstressregs2':
1530         case 'jitstress2_jitstressregs3':
1531         case 'jitstress2_jitstressregs4':
1532         case 'jitstress2_jitstressregs8':
1533         case 'jitstress2_jitstressregs0x10':
1534         case 'jitstress2_jitstressregs0x80':
1535         case 'jitstress2_jitstressregs0x1000':
1536         case 'tailcallstress':
1537         case 'jitsse2only':
1538         case 'jitnosimd':
1539         case 'jitnox86hwintrinsic':
1540         case 'jitincompletehwintrinsic':
1541         case 'jitx86hwintrinsicnoavx':
1542         case 'jitx86hwintrinsicnoavx2':
1543         case 'jitx86hwintrinsicnosimd':
1544         case 'corefx_baseline':
1545         case 'corefx_minopts':
1546         case 'corefx_tieredcompilation':
1547         case 'corefx_jitstress1':
1548         case 'corefx_jitstress2':
1549         case 'corefx_jitstressregs1':
1550         case 'corefx_jitstressregs2':
1551         case 'corefx_jitstressregs3':
1552         case 'corefx_jitstressregs4':
1553         case 'corefx_jitstressregs8':
1554         case 'corefx_jitstressregs0x10':
1555         case 'corefx_jitstressregs0x80':
1556         case 'corefx_jitstressregs0x1000':
1557         case 'zapdisable':
1558             if (os == 'CentOS7.1') {
1559                 break
1560             }
1561             if (os in bidailyCrossList) {
1562                 break
1563             }
1564             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1565             // build for the flow job. We don't need cron jobs for these. Note that the
1566             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1567             // function very early, so only non-Windows gets here.
1568             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1569                 break
1570             }
1571             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1572                 break
1573             }
1574             if (jobRequiresLimitedHardware(architecture, os)) {
1575                 addPeriodicTriggerHelper(job, '@weekly')
1576             }
1577             else {
1578                 addPeriodicTriggerHelper(job, '@daily')
1579             }
1580             break
1581         case 'heapverify1':
1582         case 'gcstress0x3':
1583             if (os == 'CentOS7.1') {
1584                 break
1585             }
1586             if (os in bidailyCrossList) {
1587                 break
1588             }
1589             addPeriodicTriggerHelper(job, '@weekly')
1590             break
1591         case 'gcstress0xc':
1592         case 'gcstress0xc_zapdisable':
1593         case 'gcstress0xc_zapdisable_jitstress2':
1594         case 'gcstress0xc_zapdisable_heapverify1':
1595         case 'gcstress0xc_jitstress1':
1596         case 'gcstress0xc_jitstress2':
1597         case 'gcstress0xc_minopts_heapverify1':
1598             if (os == 'OSX10.12') {
1599                 // GCStress=C is currently not supported on OS X
1600                 break
1601             }
1602             if (os == 'CentOS7.1') {
1603                 break
1604             }
1605             if (os in bidailyCrossList) {
1606                 break
1607             }
1608             addPeriodicTriggerHelper(job, '@weekly')
1609             break
1610
1611         case 'illink':
1612             // Testing on other operating systems TBD
1613             assert (os == 'Windows_NT' || os == 'Ubuntu')
1614             if (architecture == 'x64' || architecture == 'x86') {
1615                 if (configuration == 'Checked') {
1616                     addPeriodicTriggerHelper(job, '@daily')
1617                 }
1618             }
1619             break
1620
1621         default:
1622             println("Unknown scenario: ${scenario}");
1623             assert false
1624             break
1625     }
1626     return
1627 }
1628
1629 // **************************
1630 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1631 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1632 // builds will do a couple extra steps.
1633 // **************************
1634
1635 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1636 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1637 // it's just used for internal builds
1638 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1639 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1640     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1641
1642     if (isWindowsBuildOnlyJob) {
1643         return
1644     }
1645
1646     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1647     // Non pull request builds.
1648     if (!isPR) {
1649         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1650         return
1651     }
1652
1653     def arm64Users = [
1654         'adityamandaleeka',
1655         'AndyAyersMS',
1656         'briansull',
1657         'BruceForstall',
1658         'CarolEidt',
1659         'davidwrighton',
1660         'echesakovMSFT',
1661         'erozenfeld',
1662         'janvorli',
1663         'jashook',
1664         'pgodeq',
1665         'RussKeldorph',
1666         'sandreenko',
1667         'sdmaclea',
1668         'swaroop-sridhar',
1669         'jkotas',
1670         'markwilkie',
1671         'weshaggard'
1672     ]
1673
1674     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1675     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1676
1677     def contextString = ""
1678     def triggerString = ""
1679     def needsTrigger = true
1680     def isDefaultTrigger = false
1681     def isArm64PrivateJob = false
1682     def scenarioString = ""
1683
1684     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1685     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1686     // some jobs where the Debug configuration just does builds, no tests.
1687     //
1688     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1689     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1690     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1691     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1692
1693     if (architecture == 'armem') {
1694         assert os == 'Tizen'
1695         architecture = 'armel'
1696     }
1697
1698     switch (architecture) {
1699         case 'x64_arm64_altjit':
1700         case 'x86_arm_altjit':
1701             // TODO: for consistency, add "Build and Test" at end.
1702             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1703             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1704             break
1705
1706         case 'armel':
1707         case 'arm':
1708         case 'arm64':
1709             contextString = "${os} ${architecture} Cross ${configuration}"
1710             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1711
1712             if (scenario == 'innerloop') {
1713                 contextString += " Innerloop"
1714                 triggerString += "\\W+Innerloop"
1715             }
1716             else {
1717                 contextString += " ${scenario}"
1718                 triggerString += "\\W+${scenario}"
1719             }
1720
1721             if (configuration == 'Debug') {
1722                 contextString += " Build"
1723                 triggerString += "\\W+Build"
1724             }
1725             else {
1726                 contextString += " Build and Test"
1727                 triggerString += "\\W+Build and Test"
1728             }
1729
1730             triggerString += ".*"
1731             break
1732
1733         default:
1734             scenarioString = getScenarioDisplayString(scenario)
1735             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1736             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1737
1738             switch (scenario) {
1739                 case 'normal':
1740                     triggerString += "\\W+Build and Test.*"
1741                     break
1742
1743                 case 'corefx_innerloop': // maintain this asymmetry
1744                     triggerString += "\\W+CoreFX Tests.*"
1745                     break
1746
1747                 default:
1748                     triggerString += "\\W+${scenario}.*"
1749                     break
1750             }
1751
1752             triggerString += ".*"
1753             break
1754     }
1755
1756     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1757     // non-flow jobs that are only used as part of flow jobs.
1758
1759     switch (architecture) {
1760         case 'x64': // editor brace matching: {
1761             if (scenario == 'formatting') {
1762                 assert configuration == 'Checked'
1763                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1764                     isDefaultTrigger = true
1765                     contextString = "${os} ${architecture} Formatting"
1766                 }
1767                 break
1768             }
1769
1770             switch (os) {
1771                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1772                 case 'Debian8.4':
1773                 case 'RHEL7.2':
1774                     if (scenario == 'innerloop') {
1775                         assert !isFlowJob
1776                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1777                         isDefaultTrigger = true
1778                         break
1779                     }
1780
1781                     // fall through
1782
1783                 case 'Fedora24':
1784                 case 'Ubuntu16.04':
1785                 case 'Ubuntu16.10':
1786                     assert !isFlowJob
1787                     assert scenario != 'innerloop'
1788                     contextString = "${os} ${architecture} ${configuration} Build"
1789                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1790                     break
1791
1792                 case 'Ubuntu':
1793                     if (scenario == 'illink') {
1794                         break
1795                     }
1796                     else if (scenario == 'corefx_innerloop') {
1797                         if (configuration == 'Checked') {
1798                             isDefaultTrigger = true
1799                         }
1800                         break
1801                     }
1802
1803                     // fall through
1804
1805                 case 'OSX10.12':
1806                     // Triggers on the non-flow jobs aren't necessary here
1807                     // Corefx testing uses non-flow jobs.
1808                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1809                         needsTrigger = false
1810                         break
1811                     }
1812                     switch (scenario) {
1813                         case 'innerloop':
1814                             isDefaultTrigger = true
1815                             break
1816
1817                         case 'no_tiered_compilation_innerloop':
1818                             if (os == 'Ubuntu') {
1819                                 isDefaultTrigger = true
1820                             }
1821                             break
1822
1823                         default:
1824                             break
1825                     }
1826                     break
1827
1828                 case 'CentOS7.1':
1829                     switch (scenario) {
1830                         case 'innerloop':
1831                             // CentOS uses checked for default PR tests while debug is build only
1832                             if (configuration == 'Debug') {
1833                                 isDefaultTrigger = true
1834                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1835                                 break
1836                             }
1837
1838                             // Make sure this is a flow job to get build and test.
1839                             if (!isFlowJob) {
1840                                 needsTrigger = false
1841                                 break
1842                             }
1843
1844                             if (configuration == 'Checked') {
1845                                 assert job.name.contains("flow")
1846                                 isDefaultTrigger = true
1847                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1848                             }
1849                             break
1850
1851                         case 'normal':
1852                             // Make sure this is a flow job to get build and test.
1853                             if (!isFlowJob) {
1854                                 needsTrigger = false
1855                                 break
1856                             }
1857                             break
1858
1859                         default:
1860                             break
1861                     }
1862                     break
1863
1864                 case 'Windows_NT':
1865                     switch (scenario) {
1866                         case 'innerloop':
1867                         case 'no_tiered_compilation_innerloop':
1868                             isDefaultTrigger = true
1869                             break
1870
1871                         case 'corefx_innerloop':
1872                             if (configuration == 'Checked' || configuration == 'Release') {
1873                                 isDefaultTrigger = true
1874                             }
1875                             break
1876
1877                         default:
1878                             break
1879                     }
1880                     break
1881
1882                 default:
1883                     println("Unknown os: ${os}");
1884                     assert false
1885                     break
1886
1887             } // switch (os)
1888
1889             break
1890         // editor brace matching: }
1891
1892         case 'armel': // editor brace matching: {
1893             job.with {
1894                 publishers {
1895                     azureVMAgentPostBuildAction {
1896                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1897                     }
1898                 }
1899             }
1900
1901             switch (os) {
1902                 case 'Tizen':
1903                     if (scenario == 'innerloop') {
1904                         if (configuration == 'Checked') {
1905                             isDefaultTrigger = true
1906                         }
1907                     }
1908                     break
1909             }
1910
1911             break
1912         // editor brace matching: }
1913
1914         case 'arm':
1915         case 'arm64': // editor brace matching: {
1916
1917             switch (os) {
1918                 case 'Ubuntu':
1919                 case 'Ubuntu16.04':
1920
1921                     // Triggers on the non-flow jobs aren't necessary
1922                     if (!isFlowJob) {
1923                         needsTrigger = false
1924                         break
1925                     }
1926
1927                     switch (scenario) {
1928                         case 'innerloop':
1929                         case 'no_tiered_compilation_innerloop':
1930                             if (configuration == 'Checked') {
1931                                 // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
1932                                 //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
1933                                 if (architecture == 'arm64') {
1934                                     break
1935                                 }
1936                                 isDefaultTrigger = true
1937                             }
1938                             break
1939                     }
1940                     break
1941
1942                 case 'Windows_NT':
1943                     assert isArmWindowsScenario(scenario)
1944
1945                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1946                     // the non-flow Build job. All others need a trigger on the flow job.
1947                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1948                     if (isFlowJob != needsFlowJobTrigger) {
1949                         needsTrigger = false
1950                         break
1951                     }
1952
1953                     switch (scenario) {
1954                         case 'innerloop':
1955                             if (configuration == 'Debug') {
1956                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1957                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1958                                 // People in arm64Users will get both this and the Checked Build and Test job.
1959                                 isDefaultTrigger = true
1960                             } else if (configuration == 'Checked') {
1961                                 isDefaultTrigger = true
1962                                 isArm64PrivateJob = true
1963                             }
1964                             break
1965                         default:
1966                             isArm64PrivateJob = true
1967                             break
1968                     }
1969                     break
1970                 default:
1971                     println("NYI os: ${os}");
1972                     assert false
1973                     break
1974             }
1975             break
1976
1977         // editor brace matching: }
1978         case 'x86': // editor brace matching: {
1979             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1980             if (os == 'Ubuntu') {
1981                 // Triggers on the non-flow jobs aren't necessary here
1982                 if (!isFlowJob) {
1983                     needsTrigger = false
1984                     break
1985                 }
1986
1987                 // on-demand only for ubuntu x86
1988                 contextString = "${os} ${architecture} ${configuration} Build"
1989                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
1990                 break
1991             }
1992             switch (scenario) {
1993                 case 'innerloop':
1994                 case 'no_tiered_compilation_innerloop':
1995                     isDefaultTrigger = true
1996                     break
1997                 default:
1998                     break
1999             }
2000             break
2001
2002         // editor brace matching: }
2003         case 'x64_arm64_altjit':
2004         case 'x86_arm_altjit':
2005             // Everything default
2006             break
2007
2008         default:
2009             println("Unknown architecture: ${architecture}");
2010             assert false
2011             break
2012     }
2013
2014     if (needsTrigger) {
2015         if (isArm64PrivateJob) {
2016             if (isDefaultTrigger) {
2017                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2018             }
2019             else {
2020                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2021             }
2022         }
2023         else {
2024             if (isDefaultTrigger) {
2025                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2026             }
2027             else {
2028                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2029             }
2030         }
2031     }
2032 }
2033
2034 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2035     def buildCommands = []
2036     def osGroup = getOSGroup(os)
2037     def lowerConfiguration = configuration.toLowerCase()
2038
2039     def priority = '1'
2040     if (isInnerloopTestScenario(scenario)) {
2041         priority = '0'
2042     }
2043
2044     def doCoreFxTesting = isCoreFxScenario(scenario)
2045
2046     // Calculate the build steps, archival, and xunit results
2047     switch (os) {
2048         case 'Windows_NT': // editor brace matching: {
2049             switch (architecture) {
2050                 case 'x64':
2051                 case 'x86':
2052                 case 'x86_arm_altjit':
2053                 case 'x64_arm64_altjit':
2054                     def arch = architecture
2055                     def buildOpts = ''
2056                     if (architecture == 'x86_arm_altjit') {
2057                         arch = 'x86'
2058                     }
2059                     else if (architecture == 'x64_arm64_altjit') {
2060                         arch = 'x64'
2061                     }
2062
2063                     if (scenario == 'formatting') {
2064                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2065                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2066                         break
2067                     }
2068
2069                     if (scenario == 'illink') {
2070                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2071                     }
2072
2073                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2074                     if ((lowerConfiguration == 'release') &&
2075                         (scenario in Constants.basicScenarios) &&
2076                         (architecture != 'x86_arm_altjit') &&
2077                         (architecture != 'x64_arm64_altjit')) {
2078
2079                         buildOpts += ' -enforcepgo'
2080                     }
2081
2082                     if (doCoreFxTesting) {
2083                         buildOpts += ' skiptests';
2084                     } else {
2085                         buildOpts += " -priority=${priority}"
2086                     }
2087
2088                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2089                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2090                     // 35 characters long.
2091
2092                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2093
2094                     if (!isBuildOnly) {
2095                         def runtestArguments = ''
2096                         def testOpts = 'collectdumps'
2097
2098                         if (isR2RScenario(scenario)) {
2099
2100                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2101                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2102                             // to cause the tests to be crossgen'ed.
2103
2104                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2105                                 testOpts += ' crossgenaltjit protononjit.dll'
2106                             } else {
2107                                 testOpts += ' crossgen'
2108                             }
2109
2110                             testOpts += ' runcrossgentests'
2111                         }
2112                         else if (scenario == 'jitdiff') {
2113                             testOpts += ' jitdisasm crossgen'
2114                         }
2115                         else if (scenario == 'ilrt') {
2116                             testOpts += ' ilasmroundtrip'
2117                         }
2118                         else if (isLongGc(scenario)) {
2119                             testOpts += " ${scenario} sequential"
2120                         }
2121                         else if (scenario == 'standalone_gc') {
2122                             testOpts += ' gcname clrgc.dll'
2123                         }
2124                         else if (scenario == 'illink') {
2125                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2126                         }
2127
2128                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2129                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2130                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2131                         // which is read by the xunit harness.)
2132                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2133                         {
2134                             def timeout = 1800000
2135                             testOpts += " timeout ${timeout}"
2136                         }
2137
2138                         // If we are running a stress mode, we should write out the set of key
2139                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2140
2141                         def envScriptPath = ''
2142                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2143                             def buildCommandsStr = ''
2144                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2145                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2146
2147                             if (isJitStressScenario(scenario)) {
2148                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2149                             }
2150                             else if (isR2RStressScenario(scenario)) {
2151                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2152                             }
2153
2154                             if (architecture == 'x86_arm_altjit') {
2155                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2156                             }
2157                             else if (architecture == 'x64_arm64_altjit') {
2158                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2159                             }
2160
2161                             envScriptFinalize(os, envScriptPath)
2162
2163                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2164                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2165                             // into a single script.
2166                             buildCommands += buildCommandsStr
2167                         }
2168                         else if (architecture == 'x86_arm_altjit') {
2169                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2170                         }
2171                         else if (architecture == 'x64_arm64_altjit') {
2172                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2173                         }
2174                         if (envScriptPath != '') {
2175                             testOpts += " TestEnv ${envScriptPath}"
2176                         }
2177
2178                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2179
2180                         if (doCoreFxTesting) {
2181                             if (scenario == 'corefx_innerloop') {
2182                                 // Create CORE_ROOT and testhost
2183                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2184                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2185
2186                                 // Archive and process (only) the test results
2187                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2188                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2189                             }
2190                             else {
2191                                 def workspaceRelativeFxRoot = "_/fx"
2192                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2193                                 def fxBranch = getFxBranch(branch)
2194
2195                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2196
2197                                 // Archive and process (only) the test results
2198                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2199                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2200
2201                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2202                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2203                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2204                             }
2205                         }
2206                         else if (isGcReliabilityFramework(scenario)) {
2207                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2208                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2209                         }
2210                         else {
2211                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2212                         }
2213                     } // end if (!isBuildOnly)
2214
2215                     if (!doCoreFxTesting) {
2216                         // Run the rest of the build
2217                         // Build the mscorlib for the other OS's
2218                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2219                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2220
2221                         if (arch == 'x64') {
2222                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2223                         }
2224
2225                         // Zip up the tests directory so that we don't use so much space/time copying
2226                         // 10s of thousands of files around.
2227                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2228
2229                         if (!isJitStressScenario(scenario)) {
2230                             // For Windows, pull full test results and test drops for x86/x64.
2231                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2232                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2233                         }
2234
2235                         if (scenario == 'jitdiff') {
2236                             // retrieve jit-dasm output for base commit, and run jit-diff
2237                             if (!isBuildOnly) {
2238                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2239                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2240                             }
2241                         }
2242
2243                         if (!isBuildOnly) {
2244                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2245                         }
2246                     }
2247                     break
2248                 case 'arm':
2249                 case 'arm64':
2250                     assert isArmWindowsScenario(scenario)
2251
2252                     def buildOpts = ''
2253
2254                     if (doCoreFxTesting) {
2255                         buildOpts += ' skiptests'
2256                     } else {
2257                         buildOpts += " -priority=${priority}"
2258                     }
2259
2260                     // This is now a build only job. Do not run tests. Use the flow job.
2261                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2262
2263                     if (doCoreFxTesting) {
2264                         assert isBuildOnly
2265
2266                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2267                         def envScriptPath = ''
2268                         def buildCommandsStr = ''
2269                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2270                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2271                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2272                         envScriptFinalize(os, envScriptPath)
2273                         buildCommands += buildCommandsStr
2274
2275                         def workspaceRelativeFxRootLinux = "_/fx"
2276                         def workspaceRelativeFxRootWin = "_\\fx"
2277                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2278                         def fxBranch = getFxBranch(branch)
2279
2280                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2281
2282                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2283                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2284                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2285
2286                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2287                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2288                     } else {
2289                         // Zip up the tests directory so that we don't use so much space/time copying
2290                         // 10s of thousands of files around.
2291                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2292
2293                         // Add archival.
2294                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2295                     }
2296                     break
2297                 default:
2298                     println("Unknown architecture: ${architecture}");
2299                     assert false
2300                     break
2301             }
2302             break
2303         // end case 'Windows_NT'; editor brace matching: }
2304         case 'Ubuntu':
2305         case 'Ubuntu16.04':
2306         case 'Ubuntu16.10':
2307         case 'Debian8.4':
2308         case 'OSX10.12':
2309         case 'CentOS7.1':
2310         case 'RHEL7.2':
2311         case 'Tizen':
2312         case 'Fedora24': // editor brace matching: {
2313             switch (architecture) {
2314                 case 'x64':
2315                 case 'x86':
2316                     if (architecture == 'x86' && os == 'Ubuntu') {
2317                         // build and PAL test
2318                         def dockerImage = getDockerImageName(architecture, os, true)
2319                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2320                         dockerImage = getDockerImageName(architecture, os, false)
2321                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2322                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2323                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2324                         break
2325                     }
2326
2327                     if (scenario == 'formatting') {
2328                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2329                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2330                         break
2331                     }
2332
2333                     if (scenario == 'illink') {
2334                         assert(os == 'Ubuntu')
2335                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2336                     }
2337
2338                     if (!doCoreFxTesting) {
2339                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2340                         // only on supported OS platforms.
2341                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2342                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2343
2344                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2345                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2346
2347                         // Basic archiving of the build
2348                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2349                         // And pal tests
2350                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2351                     }
2352                     else {
2353                         if (scenario == 'corefx_innerloop') {
2354                             assert os == 'Ubuntu' || 'OSX10.12'
2355                             assert architecture == 'x64'
2356
2357                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2358                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2359                             buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2360
2361                             break
2362                             // Archive and process (only) the test results
2363                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2364                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2365                         }
2366                         else {
2367                             // Corefx stress testing
2368                             assert os == 'Ubuntu'
2369                             assert architecture == 'x64'
2370                             assert lowerConfiguration == 'checked'
2371                             assert isJitStressScenario(scenario)
2372
2373                             // Build coreclr
2374                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2375
2376                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2377
2378                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2379                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2380                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2381                             buildCommands += envScriptCmds
2382
2383                             // Build and text corefx
2384                             def workspaceRelativeFxRoot = "_/fx"
2385                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2386                             def fxBranch = getFxBranch(branch)
2387
2388                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2389
2390                             // Archive and process (only) the test results
2391                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2392                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2393                         }
2394                     }
2395                     break
2396                 case 'armem':
2397                     // Emulator cross builds for ARM runs on Tizen currently
2398                     assert os == 'Tizen'
2399
2400                     def arm_abi = "armel"
2401                     def linuxCodeName = "tizen"
2402
2403                     // Unzip the Windows test binaries first. Exit with 0
2404                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2405
2406                     // Unpack the corefx binaries
2407                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2408                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2409
2410                     // Call the ARM CI script to cross build and test using docker
2411                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2412                     --mode=docker \\
2413                     --${arm_abi} \\
2414                     --linuxCodeName=${linuxCodeName} \\
2415                     --buildConfig=${lowerConfiguration} \\
2416                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2417                     --coreFxBinDir=./bin/CoreFxBinDir \\
2418                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2419
2420                     // Basic archiving of the build, no pal tests
2421                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2422                     break
2423                 case 'arm64':
2424                 case 'arm':
2425                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2426                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2427
2428                     // Add some useful information to the log file. Ignore return codes.
2429                     buildCommands += "uname -a || true"
2430
2431                     def additionalOpts = ""
2432                     if (architecture == 'arm') {
2433                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2434                     }
2435
2436                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2437                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2438                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2439                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2440                     // ZIP up the generated CoreFX runtime and tests.
2441
2442                     def dockerImage = getDockerImageName(architecture, os, true)
2443                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2444
2445                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2446
2447                     if (doCoreFxTesting) {
2448                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2449
2450                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2451                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2452                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2453                         buildCommands += envScriptCmds
2454
2455                         // Build and text corefx
2456                         def workspaceRelativeFxRootLinux = "_/fx"
2457                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2458                         def fxBranch = getFxBranch(branch)
2459
2460                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2461
2462                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2463                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2464                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2465
2466                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2467                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2468                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2469                     }
2470                     else if (isCrossGenComparisonScenario(scenario)) {
2471                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2472
2473                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2474                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2475                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2476                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2477                         def workspaceRelativeResultsDir = "_"
2478                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2479                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2480                         getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
2481                             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2482                             def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2483
2484                             buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2485                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2486                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2487                         } // crossArch
2488                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2489                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2490                     }
2491                     else if (architecture == 'arm') {
2492                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2493                         // download the appropriate CoreFX packages.
2494                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2495                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2496
2497                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2498
2499                         // ZIP up for the test job (created in the flow job code):
2500                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2501                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2502                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2503                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2504
2505                         // These commands are assumed to be run from the root of the workspace.
2506                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2507                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2508
2509                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2510                     }
2511                     else {
2512                         assert architecture == 'arm64'
2513
2514                         // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2515                         // Linux/arm64 does not use Windows-built tests.
2516
2517                         def testBuildOpts = ""
2518                         if (priority == '1') {
2519                             testBuildOpts = "priority1"
2520                         }
2521
2522                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2523
2524                         // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2525                         buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
2526
2527                         // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
2528                         // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
2529                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2530
2531                         Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2532                     }
2533
2534                     // Archive the build logs from both product and test builds.
2535                     Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
2536
2537                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2538                     // the cleanup task in Jenkins can't remove.
2539                     newJob.with {
2540                         publishers {
2541                             azureVMAgentPostBuildAction {
2542                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2543                             }
2544                         }
2545                     }
2546                     break
2547                 default:
2548                     println("Unknown architecture: ${architecture}");
2549                     assert false
2550                     break
2551             }
2552             break
2553         // editor brace matching: }
2554         default:
2555             println("Unknown os: ${os}");
2556             assert false
2557             break
2558     } // os
2559
2560     return buildCommands
2561 }
2562
2563 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2564 // Returns true if the job should be generated.
2565 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2566 {
2567     // The various "innerloop" jobs are only available as PR triggered.
2568
2569     if (!isPR) {
2570         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
2571             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
2572             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
2573             return true
2574         }
2575
2576         if (isInnerloopTestScenario(scenario)) {
2577             return false
2578         }
2579
2580         if (scenario == 'corefx_innerloop') {
2581             return false
2582         }
2583     }
2584
2585     // Tizen is only supported for armem architecture
2586     if (os == 'Tizen' && architecture != 'armem') {
2587         return false
2588     }
2589
2590     // Filter based on architecture.
2591
2592     switch (architecture) {
2593         case 'arm':
2594             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2595                 return false
2596             }
2597             break
2598         case 'arm64':
2599             if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2600                 return false
2601             }
2602             break
2603         case 'armem':
2604             if (os != 'Tizen') {
2605                 return false
2606             }
2607             break
2608         case 'x86_arm_altjit':
2609         case 'x64_arm64_altjit':
2610             if (os != 'Windows_NT') {
2611                 return false
2612             }
2613             break
2614         case 'x86':
2615             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2616                 return false
2617             }
2618             break
2619         case 'x64':
2620             // Everything implemented
2621             break
2622         default:
2623             println("Unknown architecture: ${architecture}")
2624             assert false
2625             break
2626     }
2627
2628     // Which (Windows) build only jobs are required?
2629
2630     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2631
2632     if (isBuildOnly) {
2633         switch (architecture) {
2634             case 'arm':
2635             case 'arm64':
2636                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2637                 if (!isCoreFxScenario(scenario)) {
2638                     return false
2639                 }
2640                 break
2641             case 'x64':
2642             case 'x86':
2643                 if (!isNormalOrInnerloop) {
2644                     return false
2645                 }
2646                 break
2647             default:
2648                 return false
2649         }
2650     }
2651
2652     // Filter based on scenario.
2653
2654     if (isJitStressScenario(scenario)) {
2655         if (configuration != 'Checked') {
2656             return false
2657         }
2658
2659         def isEnabledOS = (os == 'Windows_NT') ||
2660                           (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2661                           (os == 'Ubuntu' && architecture == 'arm') ||
2662                           (os == 'Ubuntu16.04' && architecture == 'arm64')
2663         if (!isEnabledOS) {
2664             return false
2665         }
2666
2667         switch (architecture) {
2668             case 'x64':
2669             case 'x86_arm_altjit':
2670             case 'x64_arm64_altjit':
2671                 break
2672
2673             case 'x86':
2674                 // x86 ubuntu: no stress modes
2675                 if (os == 'Ubuntu') {
2676                     return false
2677                 }
2678                 break
2679
2680             case 'arm':
2681             case 'arm64':
2682                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2683                 // No "regular" Windows arm corefx jobs, e.g.
2684                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2685                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2686                 if (os == 'Windows_NT') {
2687                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2688                         return false
2689                     }
2690                 }
2691                 else {
2692                     if (!isCoreFxScenario(scenario)) {
2693                         return false
2694                     }
2695                 }
2696                 break
2697
2698             default:
2699                 // armem: no stress jobs for ARM emulator.
2700                 return false
2701         }
2702     }
2703     else if (isR2RScenario(scenario)) {
2704         if (os != 'Windows_NT') {
2705             return false
2706         }
2707
2708         if (isR2RBaselineScenario(scenario)) {
2709             // no need for Debug scenario; Checked is sufficient
2710             if (configuration != 'Checked' && configuration != 'Release') {
2711                 return false
2712             }
2713         }
2714         else if (isR2RStressScenario(scenario)) {
2715             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2716             if (configuration != 'Checked') {
2717                 return false
2718             }
2719         }
2720
2721         switch (architecture) {
2722             case 'arm':
2723             case 'arm64':
2724                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2725                 return false
2726
2727             default:
2728                 break
2729         }
2730     }
2731     else if (isCrossGenComparisonScenario(scenario)) {
2732         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2733     }
2734     else {
2735         // Skip scenarios
2736         switch (scenario) {
2737             case 'ilrt':
2738                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2739                 // the default scenario build
2740                 if (os != 'Windows_NT') {
2741                     return false
2742                 }
2743                 // Only x64 for now
2744                 if (architecture != 'x64') {
2745                     return false
2746                 }
2747                 // Release only
2748                 if (configuration != 'Release') {
2749                     return false
2750                 }
2751                 break
2752             case 'jitdiff':
2753                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2754                     return false
2755                 }
2756                 if (architecture != 'x64') {
2757                     return false
2758                 }
2759                 if (configuration != 'Checked') {
2760                     return false
2761                 }
2762                 break
2763             case 'longgc':
2764             case 'gcsimulator':
2765                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2766                     return false
2767                 }
2768                 if (architecture != 'x64') {
2769                     return false
2770                 }
2771                 if (configuration != 'Release') {
2772                     return false
2773                 }
2774                 break
2775             case 'gc_reliability_framework':
2776             case 'standalone_gc':
2777                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2778                     return false
2779                 }
2780
2781                 if (architecture != 'x64') {
2782                     return false
2783                 }
2784
2785                 if (configuration != 'Release' && configuration != 'Checked') {
2786                     return false
2787                 }
2788                 break
2789             // We only run Windows and Ubuntu x64 Checked for formatting right now
2790             case 'formatting':
2791                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2792                     return false
2793                 }
2794                 if (architecture != 'x64') {
2795                     return false
2796                 }
2797                 if (configuration != 'Checked') {
2798                     return false
2799                 }
2800                 break
2801             case 'illink':
2802                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2803                     return false
2804                 }
2805                 if (architecture != 'x64' && architecture != 'x86') {
2806                     return false
2807                 }
2808                 break
2809             case 'normal':
2810                 // Nothing skipped
2811                 break
2812             case 'innerloop':
2813                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2814                     return false
2815                 }
2816                 break
2817             case 'corefx_innerloop':
2818                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2819                     return false
2820                 }
2821                 if (architecture != 'x64') {
2822                     return false
2823                 }
2824                 break
2825             default:
2826                 println("Unknown scenario: ${scenario}")
2827                 assert false
2828                 break
2829         }
2830     }
2831
2832     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2833     // runtime behavior, not compile-time behavior, are not interesting.
2834     switch (architecture) {
2835         case 'x86_arm_altjit':
2836         case 'x64_arm64_altjit':
2837             if (isGCStressRelatedTesting(scenario)) {
2838                 return false
2839             }
2840             break
2841         default:
2842             break
2843     }
2844
2845     // The job was not filtered out, so we should generate it!
2846     return true
2847 }
2848
2849 Constants.allScenarios.each { scenario ->
2850     [true, false].each { isPR ->
2851         Constants.architectureList.each { architecture ->
2852             Constants.configurationList.each { configuration ->
2853                 Constants.osList.each { os ->
2854                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2855                     // and reset the os to Windows_NT
2856                     def isBuildOnly = false
2857                     if (os == 'Windows_NT_BuildOnly') {
2858                         isBuildOnly = true
2859                         os = 'Windows_NT'
2860                     }
2861
2862                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2863                         return
2864                     }
2865
2866                     // Calculate names
2867                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2868                     def folderName = getJobFolder(scenario)
2869
2870                     // Create the new job
2871                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2872
2873                     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
2874                     // "top-level" jobs. Build only jobs are such jobs.
2875                     if (!isBuildOnly)
2876                     {
2877                         addToViews(newJob, isPR, architecture, os)
2878                     }
2879
2880                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2881
2882                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2883                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2884                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2885
2886                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2887                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2888                     // artifacts produced.
2889                     if ((architecture == 'armem') && (os == 'Tizen')) {
2890                         // Define the Windows Tests and Corefx build job names
2891                         def lowerConfiguration = configuration.toLowerCase()
2892                         def WindowsTestsName = projectFolder + '/' +
2893                                                Utilities.getFullJobName(project,
2894                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2895                                                                         false)
2896                         def fxBranch = getFxBranch(branch)
2897                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2898                                            Utilities.getFolderName(fxBranch)
2899
2900                         def arm_abi = 'armel'
2901                         def corefx_os = 'tizen'
2902
2903                         // Let's use release CoreFX to test checked CoreCLR,
2904                         // because we do not generate checked CoreFX in CoreFX CI yet.
2905                         def corefx_lowerConfiguration = lowerConfiguration
2906                         if (lowerConfiguration == 'checked') {
2907                             corefx_lowerConfiguration = 'release'
2908                         }
2909
2910                         // Copy the Windows test binaries and the Corefx build binaries
2911                         newJob.with {
2912                             steps {
2913                                 copyArtifacts(WindowsTestsName) {
2914                                     includePatterns('bin/tests/tests.zip')
2915                                     buildSelector {
2916                                         latestSuccessful(true)
2917                                     }
2918                                 }
2919                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2920                                     includePatterns('bin/build.tar.gz')
2921                                     buildSelector {
2922                                         latestSuccessful(true)
2923                                     }
2924                                 }
2925                             } // steps
2926                         } // newJob.with
2927                     }
2928
2929                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2930
2931                     newJob.with {
2932                         steps {
2933                             if (os == 'Windows_NT') {
2934                                 buildCommands.each { buildCommand ->
2935                                     batchFile(buildCommand)
2936                                 }
2937                             }
2938                             else {
2939                                 buildCommands.each { buildCommand ->
2940                                     shell(buildCommand)
2941                                 }
2942                             }
2943                         } // steps
2944                     } // newJob.with
2945
2946                 } // os
2947             } // configuration
2948         } // architecture
2949     } // isPR
2950 } // scenario
2951
2952 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2953 // Returns the newly created job.
2954 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2955 {
2956     def osGroup = getOSGroup(os)
2957     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2958
2959     def jobFolder = getJobFolder(scenario)
2960     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2961         parameters {
2962             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2963         }
2964
2965         steps {
2966             // Set up the copies
2967
2968             // Coreclr build we are trying to test
2969             //
2970             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2971
2972             copyArtifacts(inputCoreCLRBuildName) {
2973                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2974                 buildSelector {
2975                     buildNumber('${CORECLR_BUILD}')
2976                 }
2977             }
2978
2979             if (isCoreFxScenario(scenario)) {
2980
2981                 // Only arm/arm64 supported for corefx testing now.
2982                 assert architecture == 'arm' || architecture == 'arm64'
2983
2984                 // Unzip CoreFx runtime
2985                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
2986
2987                 // Unzip CoreFx tests.
2988                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
2989
2990                 // Add the script to run the corefx tests
2991                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
2992                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2993                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
2994                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
2995
2996             } else { // !isCoreFxScenario(scenario)
2997
2998                 // Unzip tests.
2999                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3000
3001                 def buildCommands = ""
3002
3003                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3004                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3005                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3006
3007                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3008                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3009
3010                 // For all jobs
3011                 addEnvVariable("CORE_ROOT", coreRootLocation)
3012                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3013                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3014
3015                 // If we are running a stress mode, we'll set those variables as well
3016                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3017                     def stressValues = null
3018                     if (isJitStressScenario(scenario)) {
3019                         stressValues = Constants.jitStressModeScenarios[scenario]
3020                     }
3021                     else {
3022                         stressValues = Constants.r2rStressScenarios[scenario]
3023                     }
3024
3025                     stressValues.each { key, value ->
3026                         addEnvVariable(key, value)
3027                     }
3028                 }
3029
3030                 if (isR2RScenario(scenario)) {
3031                     // Crossgen the framework assemblies.
3032                     buildCommands += """
3033 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3034 @goto skip_PrecompileAssembly
3035
3036 :PrecompileAssembly
3037 @REM Skip mscorlib since it is already precompiled.
3038 @if /I "%3" == "mscorlib.dll" exit /b 0
3039 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3040
3041 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3042 @if "%errorlevel%" == "-2146230517" (
3043     echo %2 is not a managed assembly.
3044 ) else if "%errorlevel%" == "-2146234344" (
3045     echo %2 is not a managed assembly.
3046 ) else if %errorlevel% neq 0 (
3047     echo Unable to precompile %2
3048 ) else (
3049     echo Precompiled %2
3050 )
3051 @exit /b 0
3052
3053 :skip_PrecompileAssembly
3054 """
3055
3056                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3057                     // crossgen on tests before running them.
3058                     addEnvVariable("RunCrossGen", "true")
3059                 } // isR2RScenario(scenario)
3060
3061                 // Create the smarty command
3062                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3063                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3064                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3065                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3066
3067                 // Exclude tests based on scenario.
3068                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3069                     addArchSpecificExclude(architecture, excludeTag)
3070                 }
3071
3072                 if (isInnerloopTestScenario(scenario)) {
3073                     addExclude("pri1")
3074                 }
3075
3076                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3077                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3078                 addExclude("LONG_RUNNING")
3079
3080                 smartyCommand += "/lstFile Tests.lst"
3081
3082                 def testListArch = [
3083                     'arm64': 'arm64',
3084                     'arm': 'arm'
3085                 ]
3086
3087                 def archLocation = testListArch[architecture]
3088
3089                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3090                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3091                 addCommand("${smartyCommand}")
3092
3093                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3094                 // However, we also need to remove all the variables that were set during this batch file, so we
3095                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3096                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3097                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3098                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3099                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3100                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3101
3102                 // ZIP up the smarty output, no matter what the smarty result.
3103                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3104
3105                 addCommand("echo %errorlevel%")
3106                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3107
3108                 // Use the smarty errorlevel as the script errorlevel.
3109                 addCommand("exit /b %__save_smarty_errorlevel%")
3110
3111                 batchFile(buildCommands)
3112             } // non-corefx testing
3113         } // steps
3114     } // job
3115
3116     if (!isCoreFxScenario(scenario)) {
3117         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3118
3119         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3120         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3121         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3122         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3123     }
3124
3125     return newJob
3126 }
3127
3128 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3129 // E.g., non-Windows tests.
3130 // Returns the newly created job.
3131 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3132 {
3133     def lowerConfiguration = configuration.toLowerCase()
3134
3135     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3136     def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3137     def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3138
3139     def doCoreFxTesting = isCoreFxScenario(scenario)
3140
3141     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3142
3143     def osGroup = getOSGroup(os)
3144     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3145
3146     def testOpts = ''
3147     def useServerGC = false
3148
3149     // Enable Server GC for Ubuntu PR builds
3150     // REVIEW: why? Does this apply to all architectures? Why only PR?
3151     if (os == 'Ubuntu' && isPR) {
3152         testOpts += ' --useServerGC'
3153         useServerGC = true
3154     }
3155
3156     if (isR2RScenario(scenario)) {
3157
3158         testOpts += ' --crossgen --runcrossgentests'
3159
3160         if (scenario == 'r2r_jitstress1') {
3161             testOpts += ' --jitstress=1'
3162         }
3163         else if (scenario == 'r2r_jitstress2') {
3164             testOpts += ' --jitstress=2'
3165         }
3166         else if (scenario == 'r2r_jitstress1_tiered') {
3167             testOpts += ' --jitstress=1'
3168         }
3169         else if (scenario == 'r2r_jitstress2_tiered') {
3170             testOpts += ' --jitstress=2'
3171         }
3172         else if (scenario == 'r2r_jitstressregs1') {
3173             testOpts += ' --jitstressregs=1'
3174         }
3175         else if (scenario == 'r2r_jitstressregs2') {
3176             testOpts += ' --jitstressregs=2'
3177         }
3178         else if (scenario == 'r2r_jitstressregs3') {
3179             testOpts += ' --jitstressregs=3'
3180         }
3181         else if (scenario == 'r2r_jitstressregs4') {
3182             testOpts += ' --jitstressregs=4'
3183         }
3184         else if (scenario == 'r2r_jitstressregs8') {
3185             testOpts += ' --jitstressregs=8'
3186         }
3187         else if (scenario == 'r2r_jitstressregs0x10') {
3188             testOpts += ' --jitstressregs=0x10'
3189         }
3190         else if (scenario == 'r2r_jitstressregs0x80') {
3191             testOpts += ' --jitstressregs=0x80'
3192         }
3193         else if (scenario == 'r2r_jitstressregs0x1000') {
3194             testOpts += ' --jitstressregs=0x1000'
3195         }
3196         else if (scenario == 'r2r_jitminopts') {
3197             testOpts += ' --jitminopts'
3198         }
3199         else if (scenario == 'r2r_jitforcerelocs') {
3200             testOpts += ' --jitforcerelocs'
3201         }
3202         else if (scenario == 'r2r_gcstress15') {
3203             testOpts += ' --gcstresslevel=0xF'
3204         }
3205     }
3206     else if (scenario == 'jitdiff') {
3207         testOpts += ' --jitdisasm --crossgen'
3208     }
3209     else if (scenario == 'illink') {
3210         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3211     }
3212     else if (isLongGc(scenario)) {
3213         // Long GC tests behave very poorly when they are not
3214         // the only test running (many of them allocate until OOM).
3215         testOpts += ' --sequential'
3216
3217         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3218         // for running long GC and GCSimulator tests, respectively. We don't use them
3219         // here because using a playlist file produces much more readable output on the CI machines
3220         // and reduces running time.
3221         //
3222         // The Long GC playlist contains all of the tests that are
3223         // going to be run. The GCSimulator playlist contains all of
3224         // the GC simulator tests.
3225         if (scenario == 'longgc') {
3226             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3227         }
3228         else if (scenario == 'gcsimulator') {
3229             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3230         }
3231     }
3232     else if (isGcReliabilityFramework(scenario)) {
3233         testOpts += ' --build-overlay-only'
3234     }
3235     else if (scenario == 'standalone_gc') {
3236         if (osGroup == 'OSX') {
3237             testOpts += ' --gcname=libclrgc.dylib'
3238         }
3239         else if (osGroup == 'Linux') {
3240             testOpts += ' --gcname=libclrgc.so'
3241         }
3242         else {
3243             println("Unexpected OS group: ${osGroup} for os ${os}")
3244             assert false
3245         }
3246     }
3247
3248     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3249     // will be null in this case.
3250
3251     def jobFolder = getJobFolder(scenario)
3252     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3253         parameters {
3254             if (inputTestsBuildName != null) {
3255                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3256             }
3257             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3258         }
3259
3260         steps {
3261             // Set up the copies
3262
3263             // Coreclr build containing the tests and mscorlib
3264             // pri1 jobs still need to copy windows_nt built tests
3265             if (inputTestsBuildName != null) {
3266                 copyArtifacts(inputTestsBuildName) {
3267                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3268                     buildSelector {
3269                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3270                     }
3271                 }
3272             }
3273
3274             // Coreclr build we are trying to test
3275             //
3276             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3277             //
3278             // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3279             // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3280
3281             if (!isUbuntuArm64Job) {
3282                 copyArtifacts(inputCoreCLRBuildName) {
3283                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3284                     buildSelector {
3285                         buildNumber('${CORECLR_BUILD}')
3286                     }
3287                 }
3288             }
3289
3290             if (isUbuntuArmJob) {
3291                 // Add some useful information to the log file. Ignore return codes.
3292                 shell("uname -a || true")
3293             }
3294
3295             if (isUbuntuArm64Job) {
3296                 // Copy the required artifacts directly, using wget, e.g.:
3297                 //
3298                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3299                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3300                 //
3301                 // parameterized as:
3302                 //
3303                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3304                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3305                 //
3306                 // CoreFX example artifact URLs:
3307                 //
3308                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3309                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3310                 //
3311                 // Note that the source might be in a "jitstress" folder.
3312                 //
3313                 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3314                 //
3315                 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3316
3317                 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3318
3319                 def mungedProjectName = Utilities.getFolderName(project)
3320                 def mungedBranchName = Utilities.getFolderName(branch)
3321
3322                 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3323                 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3324                 if (doCoreFxTesting || doCrossGenComparison) {
3325                     // These depend on unique builds for each scenario
3326                     inputCoreCLRBuildScenario = scenario
3327                 }
3328                 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3329                 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3330
3331                 // Need to add the sub-folder if necessary.
3332                 def inputJobPath = "job/${inputJobName}"
3333                 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3334                 if (folderName != '') {
3335                     inputJobPath = "job/${folderName}/job/${inputJobName}"
3336                 }
3337
3338                 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3339
3340                 if (doCoreFxTesting) {
3341                     shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3342                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3343                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3344                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/run-test.sh")
3345                     shell("chmod +x ${workspaceRelativeFxRootLinux}/run-test.sh")
3346                 }
3347                 else {
3348                     shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
3349                     shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
3350                 }
3351             }
3352
3353             if (architecture == 'x86') {
3354                 shell("mkdir ./bin/CoreFxNative")
3355
3356                 def fxBranch = getFxBranch(branch)
3357                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3358
3359                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3360                     includePatterns('bin/build.tar.gz')
3361                     targetDirectory('bin/CoreFxNative')
3362                     buildSelector {
3363                         latestSuccessful(true)
3364                     }
3365                 }
3366
3367                 shell("mkdir ./bin/CoreFxBinDir")
3368                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3369             }
3370
3371             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3372             if (doCoreFxTesting) {
3373                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3374                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3375             }
3376             else if (architecture != 'arm64') {
3377                 // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
3378                 //
3379                 // Unzip the tests first.  Exit with 0
3380                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3381                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3382             }
3383
3384             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3385             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3386             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3387             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3388             // copied correctly.
3389             if (!doCoreFxTesting) {
3390                 if (isUbuntuArmJob) {
3391                     if (architecture == 'arm') {
3392                         shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3393                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3394                     }
3395                     else {
3396                         assert architecture == 'arm64'
3397                         shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
3398
3399                         // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
3400                         shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3401                     }
3402                 }
3403                 else {
3404                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3405                 }
3406             }
3407
3408             // Execute the tests
3409             def runDocker = isNeedDocker(architecture, os, false)
3410             def dockerPrefix = ""
3411             def dockerCmd = ""
3412             if (runDocker) {
3413                 def dockerImage = getDockerImageName(architecture, os, false)
3414                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3415                 dockerCmd = dockerPrefix + "${dockerImage} "
3416             }
3417
3418             // If we are running a stress mode, we'll set those variables first.
3419             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3420             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3421                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3422                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3423                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3424                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3425                 shell("${envScriptCmds}")
3426                 testOpts += " --test-env=${scriptFileName}"
3427             }
3428
3429             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3430             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3431             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3432             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3433             if (isGCStressRelatedTesting(scenario)) {
3434                 if (architecture == 'x64') {
3435                     shell('./init-tools.sh')
3436                 }
3437             }
3438
3439             if (doCoreFxTesting) {
3440                 shell("""\
3441 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3442             }
3443             else {
3444                 def runScript = "${dockerCmd}./tests/runtest.sh"
3445
3446                 // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
3447
3448                 shell("""\
3449 ${runScript} \\
3450     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3451     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3452     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3453     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3454             }
3455
3456             if (isGcReliabilityFramework(scenario)) {
3457                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3458                 if (useServerGC) {
3459                     if (runDocker) {
3460                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3461                     }
3462                     else {
3463                         shell("export COMPlus_gcServer=1")
3464                     }
3465                 }
3466
3467                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3468             }
3469         } // steps
3470     } // job
3471
3472     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3473     if (os in ['Ubuntu']) {
3474         SummaryBuilder summaries = new SummaryBuilder()
3475         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3476         summaries.emit(newJob)
3477     }
3478
3479     if (doCoreFxTesting) {
3480         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3481         if ((os == "Ubuntu") && (architecture == 'arm')) {
3482             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3483             //
3484             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3485             //
3486             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3487             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3488         }
3489         else {
3490             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3491         }
3492     }
3493     else {
3494         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3495     }
3496
3497     return newJob
3498 }
3499
3500 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3501 {
3502     assert isCrossGenComparisonScenario(scenario)
3503
3504     def osGroup = getOSGroup(os)
3505     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3506
3507     def workspaceRelativeResultsDir = "_"
3508     def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3509
3510     def jobFolder = getJobFolder(scenario)
3511     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3512         parameters {
3513             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3514         }
3515
3516         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3517
3518         steps {
3519             copyArtifacts(inputCoreCLRBuildName) {
3520                 includePatterns("${workspaceRelativeArtifactsArchive}")
3521                 buildSelector {
3522                     buildNumber('${CORECLR_BUILD}')
3523                 }
3524             }
3525
3526             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3527
3528             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3529             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3530             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3531             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3532
3533             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3534             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3535
3536             shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3537             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3538             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3539
3540             getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3541                 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3542                 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3543             } // crossArch
3544         } // steps
3545     }  // job
3546
3547     Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**")
3548     getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3549         def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3550         Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**")
3551     } // crossArch
3552
3553     return newJob
3554 }
3555
3556 // Create a test job that will be used by a flow job.
3557 // Returns the newly created job.
3558 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3559 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3560 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3561 {
3562     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3563
3564     def newJob = null
3565     if (windowsArmJob) {
3566         assert inputTestsBuildName == null
3567         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3568     }
3569     else if (isCrossGenComparisonScenario(scenario)) {
3570         assert inputTestsBuildName == null
3571         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3572     }
3573     else {
3574         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3575     }
3576
3577     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3578
3579     if (scenario == 'jitdiff') {
3580         def osGroup = getOSGroup(os)
3581         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3582     }
3583
3584     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3585     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3586
3587     return newJob
3588 }
3589
3590 // Create a flow job to tie together a build job with the given test job.
3591 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3592 // Returns the new flow job.
3593 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3594 {
3595     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3596     // Linux CoreCLR test
3597     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3598     def jobFolder = getJobFolder(scenario)
3599
3600     def newFlowJob = null
3601
3602     if (inputTestsBuildName == null) {
3603         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3604                         buildFlow("""\
3605 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3606
3607 // And then build the test build
3608 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3609 """)
3610         }
3611         JobReport.Report.addReference(inputCoreCLRBuildName)
3612         JobReport.Report.addReference(fullTestJobName)
3613     }
3614     else {
3615         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3616                         buildFlow("""\
3617 // Build the input jobs in parallel
3618 parallel (
3619 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3620 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3621 )
3622
3623 // And then build the test build
3624 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3625                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3626 """)
3627         }
3628         JobReport.Report.addReference(inputCoreCLRBuildName)
3629         JobReport.Report.addReference(inputTestsBuildName)
3630         JobReport.Report.addReference(fullTestJobName)
3631     }
3632
3633     addToViews(newFlowJob, isPR, architecture, os)
3634
3635     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3636
3637     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3638     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3639
3640     return newFlowJob
3641 }
3642
3643 // Determine if we should generate a flow job for the given parameters.
3644 // Returns true if the job should be generated.
3645 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3646 {
3647     // The various "innerloop" jobs are only available as PR triggered.
3648
3649     if (!isPR) {
3650         if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
3651             // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
3652             //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
3653             return true
3654         }
3655
3656         if (isInnerloopTestScenario(scenario)) {
3657             return false
3658         }
3659
3660         if (scenario == 'corefx_innerloop') {
3661             return false
3662         }
3663     }
3664
3665     // Filter based on OS and architecture.
3666
3667     switch (architecture) {
3668         case 'arm':
3669             if (os != "Ubuntu" && os != "Windows_NT") {
3670                 return false
3671             }
3672             break
3673         case 'arm64':
3674             if (os != "Ubuntu16.04" && os != "Windows_NT") {
3675                 return false
3676             }
3677             break
3678         case 'x86':
3679             if (os != "Ubuntu") {
3680                 return false
3681             }
3682             break
3683         case 'x64':
3684             if (!(os in Constants.crossList)) {
3685                 return false
3686             }
3687             if (os == "Windows_NT") {
3688                 return false
3689             }
3690             break
3691         case 'armem':
3692         case 'x86_arm_altjit':
3693         case 'x64_arm64_altjit':
3694             // No flow jobs
3695             return false
3696         default:
3697             println("Unknown architecture: ${architecture}")
3698             assert false
3699             break
3700     }
3701
3702     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3703
3704     // Filter based on scenario in OS.
3705
3706     if (os == 'Windows_NT') {
3707         assert architecture == 'arm' || architecture == 'arm64'
3708         if (!isArmWindowsScenario(scenario)) {
3709             return false
3710         }
3711         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3712             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3713             // To do that, it doesn't require a flow job.
3714             return false
3715         }
3716     }
3717     else {
3718         // Non-Windows
3719         if (architecture == 'arm') {
3720             if (!(scenario in Constants.validLinuxArmScenarios)) {
3721                 return false
3722             }
3723         }
3724         else if (architecture == 'arm64') {
3725             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3726                 return false
3727             }
3728         }
3729         else if (architecture == 'x86') {
3730             // Linux/x86 only want innerloop and default test
3731             if (!isNormalOrInnerloop) {
3732                 return false
3733             }
3734         }
3735         else if (architecture == 'x64') {
3736             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3737             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3738             // build from a Windows machine.
3739             if (isCoreFxScenario(scenario)) {
3740                 return false
3741             }
3742         }
3743     }
3744
3745     // For CentOS, we only want Checked/Release builds.
3746     if (os == 'CentOS7.1') {
3747         if (configuration != 'Checked' && configuration != 'Release') {
3748             return false
3749         }
3750         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3751             return false
3752         }
3753     }
3754
3755     // For RedHat and Debian, we only do Release builds.
3756     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3757         if (configuration != 'Release') {
3758             return false
3759         }
3760         if (!isNormalOrInnerloop) {
3761             return false
3762         }
3763     }
3764
3765     // Next, filter based on scenario.
3766
3767     if (isJitStressScenario(scenario)) {
3768         if (configuration != 'Checked') {
3769             return false
3770         }
3771     }
3772     else if (isR2RBaselineScenario(scenario)) {
3773         if (configuration != 'Checked' && configuration != 'Release') {
3774             return false
3775         }
3776     }
3777     else if (isR2RStressScenario(scenario)) {
3778         if (configuration != 'Checked') {
3779             return false
3780         }
3781     }
3782     else if (isCrossGenComparisonScenario(scenario)) {
3783         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3784     }
3785     else {
3786         // Skip scenarios
3787         switch (scenario) {
3788             case 'ilrt':
3789             case 'longgc':
3790             case 'gcsimulator':
3791                 // Long GC tests take a long time on non-Release builds
3792                 // ilrt is also Release only
3793                 if (configuration != 'Release') {
3794                     return false
3795                 }
3796                 break
3797
3798             case 'jitdiff':
3799                 if (configuration != 'Checked') {
3800                     return false
3801                 }
3802                 break
3803
3804             case 'gc_reliability_framework':
3805             case 'standalone_gc':
3806                 if (configuration != 'Release' && configuration != 'Checked') {
3807                     return false
3808                 }
3809                 break
3810
3811             case 'formatting':
3812                 return false
3813
3814             case 'illink':
3815                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3816                     return false
3817                 }
3818                 break
3819
3820             case 'normal':
3821                 // Nothing skipped
3822                 break
3823
3824             case 'innerloop':
3825                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3826                     return false
3827                 }
3828                 break
3829
3830             case 'corefx_innerloop':
3831                 // No flow job needed
3832                 return false
3833
3834             default:
3835                 println("Unknown scenario: ${scenario}")
3836                 assert false
3837                 break
3838         }
3839     }
3840
3841     // The job was not filtered out, so we should generate it!
3842     return true
3843 }
3844
3845 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3846 Constants.allScenarios.each { scenario ->
3847     [true, false].each { isPR ->
3848         Constants.architectureList.each { architecture ->
3849             Constants.configurationList.each { configuration ->
3850                 Constants.osList.each { os ->
3851
3852                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3853                         return
3854                     }
3855
3856                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3857                     def doCoreFxTesting = isCoreFxScenario(scenario)
3858                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3859
3860                     // Figure out the job name of the CoreCLR build the test will depend on.
3861
3862                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3863                     def inputCoreCLRBuildIsBuildOnly = false
3864                     if (doCoreFxTesting) {
3865                         // Every CoreFx test depends on its own unique build.
3866                         inputCoreCLRBuildScenario = scenario
3867                         if (windowsArmJob) {
3868                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3869                             inputCoreCLRBuildIsBuildOnly = true
3870                         }
3871                     }
3872                     if (doCrossGenComparison) {
3873                         inputCoreCLRBuildScenario = scenario
3874                     }
3875
3876                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3877                     def inputCoreCLRBuildName = projectFolder + '/' +
3878                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3879
3880                     // Figure out the name of the build job that the test job will depend on.
3881                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3882                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3883                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3884                     // CoreFX build creates the tests.
3885
3886                     def inputTestsBuildName = null
3887
3888                     // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
3889                     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3890
3891                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job) {
3892                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3893
3894                         def inputTestsBuildArch = architecture
3895                         if (architecture == "arm") {
3896                             // Use the x86 test build for arm unix
3897                             inputTestsBuildArch = "x86"
3898                         }
3899
3900                         def inputTestsBuildIsBuildOnly = true
3901
3902                         inputTestsBuildName = projectFolder + '/' +
3903                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3904                     }
3905
3906                     // =============================================================================================
3907                     // Create the test job
3908                     // =============================================================================================
3909
3910                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3911
3912                     // =============================================================================================
3913                     // Create a build flow to join together the build and tests required to run this test.
3914                     // =============================================================================================
3915
3916                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3917                         // Do not create the flow job for RHEL jobs.
3918                         return
3919                     }
3920
3921                     def fullTestJobName = projectFolder + '/' + testJob.name
3922                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3923
3924                 } // os
3925             } // configuration
3926         } // architecture
3927     } // isPR
3928 } // scenario
3929
3930 JobReport.Report.generateJobReport(out)
3931
3932 // Make the call to generate the help job
3933 Utilities.createHelperJob(this, project, branch,
3934     "Welcome to the ${project} Repository",  // This is prepended to the help message
3935     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3936
3937 Utilities.addCROSSCheck(this, project, branch)