netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
  39     def static LimitedHardwareBranches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'CentOS7.1',
  65                'RHEL7.2']
  66
  67     // This is a set of JIT stress modes combined with the set of variables that
  68     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  69     // the values are the environment variables
  70     def static jitStressModeScenarios = [
  71                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  72                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  73                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  74                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  75                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  76                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  77                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  78                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  79                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  80                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  81                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  82                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  83                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  84                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  85                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  86                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  87                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  88                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  89                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  90                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  91                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  92                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  93                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  94                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  95                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  96                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  97                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  98                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  99                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 100                'corefx_baseline'                : [ : ], // corefx baseline
 101                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
 102                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
 103                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
 104                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
 105                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 106                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 107                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 108                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 109                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 110                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 111                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 112                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 113                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 114                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 115                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 116                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 117                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 118                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 119                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 120                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 121                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 122                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 123     ]
 124
 125     // This is a set of ReadyToRun stress scenarios
 126     def static r2rStressScenarios = [
 127                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 128                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 129                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 130                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 131                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 132                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 133                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 134                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 135                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 136                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 137                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 138                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 139                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 140     ]
 141
 142     // This is the basic set of scenarios
 143     def static basicScenarios = [
 144                'innerloop',
 145                'normal',
 146                'ilrt',
 147                'r2r',
 148                'longgc',
 149                'formatting',
 150                'gcsimulator',
 151                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 152                'standalone_gc',
 153                'gc_reliability_framework',
 154                'illink',
 155                'corefx_innerloop']
 156
 157     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 158
 159     // Valid PR trigger combinations.
 160     def static prTriggeredValidInnerLoopCombos = [
 161         'Windows_NT': [
 162             'x64': [
 163                 'Checked'
 164             ],
 165             'x86': [
 166                 'Checked',
 167                 'Release'
 168             ],
 169             'arm': [
 170                 'Checked',
 171             ],
 172             'arm64': [
 173                 'Checked'
 174             ]
 175         ],
 176         'Windows_NT_BuildOnly': [
 177             'x64': [
 178                 'Checked',
 179                 'Release'
 180             ],
 181             'x86': [
 182                 'Checked',
 183                 'Release'
 184             ],
 185             'arm': [
 186                 'Checked'
 187             ],
 188         ],
 189         'Ubuntu': [
 190             'x64': [
 191                 'Checked'
 192             ],
 193             'arm64': [
 194                 'Debug'
 195             ],
 196             'arm': [
 197                 'Checked'
 198             ]
 199         ],
 200         'CentOS7.1': [
 201             'x64': [
 202                 'Debug',
 203                 'Checked'
 204             ]
 205         ],
 206         'OSX10.12': [
 207             'x64': [
 208                 'Checked'
 209             ]
 210         ],
 211         'Tizen': [
 212             'armem': [
 213                 'Checked'
 214             ]
 215         ],
 216     ]
 217
 218     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 219     // to Tests.lst file categories to exclude.
 220     //
 221     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 222     // and with the same values, with some commented out, for easier maintenance.
 223     //
 224     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 225     //
 226     def static validArmWindowsScenarios = [
 227                'innerloop':                              [],
 228                'normal':                                 [],
 229                // 'ilrt'
 230                'r2r':                                    ["R2R_FAIL", "R2R_EXCLUDE"],
 231                // 'longgc'
 232                // 'formatting'
 233                // 'gcsimulator'
 234                // 'jitdiff'
 235                // 'standalone_gc'
 236                // 'gc_reliability_framework'
 237                // 'illink'
 238                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'r2r_jitstressregs1':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'r2r_jitstressregs2':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'r2r_jitstressregs3':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'r2r_jitstressregs4':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'r2r_jitstressregs8':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'r2r_jitminopts':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 249                'r2r_jitforcerelocs':                     ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'r2r_gcstress15':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 251                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 252                'tieredcompilation':                      [],
 253                'forcerelocs':                            [],
 254                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 255                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 256                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 257                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 258                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 259                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 260                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 261                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 262                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 263                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 264                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 265                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 266                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 267                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 268                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 269                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 270                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 271                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 272                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 273                // 'jitsse2only'                          // Only relevant to xarch
 274                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 275                // 'jitincompletehwintrinsic'
 276                // 'jitx86hwintrinsicnoavx'
 277                // 'jitx86hwintrinsicnoavx2'
 278                // 'jitx86hwintrinsicnosimd'
 279                // 'jitnox86hwintrinsic'
 280                'corefx_baseline':                        [],    // corefx tests don't use smarty
 281                'corefx_minopts':                         [],    // corefx tests don't use smarty
 282                'corefx_tieredcompilation':               [],    // corefx tests don't use smarty
 283                'corefx_jitstress1':                      [],    // corefx tests don't use smarty
 284                'corefx_jitstress2':                      [],    // corefx tests don't use smarty
 285                'corefx_jitstressregs1':                  [],    // corefx tests don't use smarty
 286                'corefx_jitstressregs2':                  [],    // corefx tests don't use smarty
 287                'corefx_jitstressregs3':                  [],    // corefx tests don't use smarty
 288                'corefx_jitstressregs4':                  [],    // corefx tests don't use smarty
 289                'corefx_jitstressregs8':                  [],    // corefx tests don't use smarty
 290                'corefx_jitstressregs0x10':               [],    // corefx tests don't use smarty
 291                'corefx_jitstressregs0x80':               [],    // corefx tests don't use smarty
 292                'corefx_jitstressregs0x1000':             [],    // corefx tests don't use smarty
 293                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 294                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 295                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 296                'heapverify1':                            [],
 297                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 298                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 299                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 300                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 301                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 302                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 303
 304                //
 305                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 306                //
 307
 308                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 309                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 310                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 311                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 312                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 313                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 314                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 315                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 316                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 317                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 318                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 319                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 320                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 321                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 322                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 323                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 324                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 325                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 326                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 327     ]
 328
 329     def static validLinuxArmScenarios = [
 330                'innerloop',
 331                'normal',
 332                // 'ilrt'
 333                'r2r',
 334                // 'longgc'
 335                // 'formatting'
 336                // 'gcsimulator'
 337                // 'jitdiff'
 338                // 'standalone_gc'
 339                // 'gc_reliability_framework'
 340                // 'illink'
 341                'r2r_jitstress1',
 342                'r2r_jitstress2',
 343                'r2r_jitstressregs1',
 344                'r2r_jitstressregs2',
 345                'r2r_jitstressregs3',
 346                'r2r_jitstressregs4',
 347                'r2r_jitstressregs8',
 348                'r2r_jitstressregs0x10',
 349                'r2r_jitstressregs0x80',
 350                'r2r_jitstressregs0x1000',
 351                'r2r_jitminopts',
 352                'r2r_jitforcerelocs',
 353                'r2r_gcstress15',
 354                'minopts',
 355                'tieredcompilation',
 356                'forcerelocs',
 357                'jitstress1',
 358                'jitstress2',
 359                'jitstressregs1',
 360                'jitstressregs2',
 361                'jitstressregs3',
 362                'jitstressregs4',
 363                'jitstressregs8',
 364                'jitstressregs0x10',
 365                'jitstressregs0x80',
 366                'jitstressregs0x1000',
 367                'jitstress2_jitstressregs1',
 368                'jitstress2_jitstressregs2',
 369                'jitstress2_jitstressregs3',
 370                'jitstress2_jitstressregs4',
 371                'jitstress2_jitstressregs8',
 372                'jitstress2_jitstressregs0x10',
 373                'jitstress2_jitstressregs0x80',
 374                'jitstress2_jitstressregs0x1000',
 375                'tailcallstress',
 376                // 'jitsse2only'                          // Only relevant to xarch
 377                // 'jitnosimd'
 378                // 'jitincompletehwintrinsic'
 379                // 'jitx86hwintrinsicnoavx'
 380                // 'jitx86hwintrinsicnoavx2'
 381                // 'jitx86hwintrinsicnosimd'
 382                // 'jitnox86hwintrinsic'
 383                'corefx_baseline',
 384                'corefx_minopts',
 385                'corefx_tieredcompilation',
 386                'corefx_jitstress1',
 387                'corefx_jitstress2',
 388                'corefx_jitstressregs1',
 389                'corefx_jitstressregs2',
 390                'corefx_jitstressregs3',
 391                'corefx_jitstressregs4',
 392                'corefx_jitstressregs8',
 393                'corefx_jitstressregs0x10',
 394                'corefx_jitstressregs0x80',
 395                'corefx_jitstressregs0x1000',
 396                'gcstress0x3',
 397                'gcstress0xc',
 398                'zapdisable',
 399                'heapverify1',
 400                'gcstress0xc_zapdisable',
 401                'gcstress0xc_zapdisable_jitstress2',
 402                'gcstress0xc_zapdisable_heapverify1',
 403                'gcstress0xc_jitstress1',
 404                'gcstress0xc_jitstress2',
 405                'gcstress0xc_minopts_heapverify1'
 406     ]
 407
 408     def static configurationList = ['Debug', 'Checked', 'Release']
 409
 410     // This is the set of architectures
 411     // Some of these are pseudo-architectures:
 412     //    armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs.
 413     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 414     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 415     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 416
 417     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 418     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 419 }
 420
 421 // **************************************************************
 422 // Create some specific views
 423 //
 424 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 425 // views based on a single regular expression. These views will be generated by adding a
 426 // specific set of jobs to them.
 427 //
 428 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 429 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 430 // those are required, don't add them (which simplifies the view pages, as well).
 431 // **************************************************************
 432
 433 class Views {
 434     def static MergeJobView = null
 435     def static PeriodicJobView = null
 436     def static ArchitectureViews = [:]
 437     def static OSViews = [:]
 438 }
 439
 440 // MergeJobView: include all jobs that execute when a PR change is merged.
 441 Views.MergeJobView = listView('Merge') {
 442     recurse()
 443     columns {
 444         status()
 445         weather()
 446         name()
 447         lastSuccess()
 448         lastFailure()
 449         lastDuration()
 450         buildButton()
 451     }
 452 }
 453
 454 // PeriodicJobView: include all jobs that execute on a schedule
 455 Views.PeriodicJobView = listView('Periodic') {
 456     recurse()
 457     columns {
 458         status()
 459         weather()
 460         name()
 461         lastSuccess()
 462         lastFailure()
 463         lastDuration()
 464         buildButton()
 465     }
 466 }
 467
 468 // Create a view for non-PR jobs for each architecture.
 469 Constants.architectureList.each { architecture ->
 470     Views.ArchitectureViews[architecture] = listView(architecture) {
 471         recurse()
 472         columns {
 473             status()
 474             weather()
 475             name()
 476             lastSuccess()
 477             lastFailure()
 478             lastDuration()
 479             buildButton()
 480         }
 481     }
 482 }
 483
 484 // Create a view for non-PR jobs for each OS.
 485 Constants.osList.each { os ->
 486     // Don't create one for the special 'Windows_NT_BuildOnly'
 487     if (os == 'Windows_NT_BuildOnly') {
 488         return
 489     }
 490     Views.OSViews[os] = listView(os) {
 491         recurse()
 492         columns {
 493             status()
 494             weather()
 495             name()
 496             lastSuccess()
 497             lastFailure()
 498             lastDuration()
 499             buildButton()
 500         }
 501     }
 502 }
 503
 504 def static addToMergeView(def job) {
 505     Views.MergeJobView.with {
 506         jobs {
 507             name(job.name)
 508         }
 509     }
 510 }
 511
 512 def static addToPeriodicView(def job) {
 513     Views.PeriodicJobView.with {
 514         jobs {
 515             name(job.name)
 516         }
 517     }
 518 }
 519
 520 def static addToViews(def job, def isPR, def architecture, def os) {
 521     if (isPR) {
 522         // No views want PR jobs currently.
 523         return
 524     }
 525
 526     // Add to architecture view.
 527     Views.ArchitectureViews[architecture].with {
 528         jobs {
 529             name(job.name)
 530         }
 531     }
 532
 533     // Add to OS view.
 534     Views.OSViews[os].with {
 535         jobs {
 536             name(job.name)
 537         }
 538     }
 539 }
 540
 541 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 542     addToPeriodicView(job)
 543     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 544 }
 545
 546 def static addGithubPushTriggerHelper(def job) {
 547     addToMergeView(job)
 548     Utilities.addGithubPushTrigger(job)
 549 }
 550
 551
 552 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 553     assert os instanceof String
 554     assert architecture instanceof String
 555
 556     def armArches = ['arm', 'armem', 'arm64']
 557     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 558
 559     if (!(architecture in armArches)) {
 560         assert options == null
 561         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 562
 563         return
 564     }
 565
 566     // This is an arm(64) job.
 567     //
 568     // There are several options.
 569     //
 570     // Windows_NT
 571     //
 572     // Arm32 (Build) -> latest-arm64
 573     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 574     // Arm32 (Test)  -> arm64-windows_nt
 575     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 576     //
 577     // Arm64 (Build) -> latest-arm64
 578     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 579     // Arm64 (Test)  -> arm64-windows_nt
 580     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 581     //
 582     // Ubuntu
 583     //
 584     // Arm32 emulator (Build, Test) -> arm-cross-latest
 585     //       |-> os in supportedArmLinuxOs && (architecture == "armem")
 586     //
 587     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 588     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 589     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 590     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 591     // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
 592     //       |-> os == "Ubuntu" && (architecture == "arm")
 593     //
 594     // Arm64 (Build) -> arm64-cross-latest
 595     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 596     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 597     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 598     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 599     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 600
 601     // This has to be a arm arch
 602     assert architecture in armArches
 603     if (os == "Windows_NT") {
 604         // arm32/arm64 Windows jobs share the same machines for now
 605         def isBuild = options['use_arm64_build_machine'] == true
 606
 607         if (isBuild == true) {
 608             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 609         } else {
 610             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 611         }
 612     } else {
 613         assert os != 'Windows_NT'
 614         assert os in supportedArmLinuxOs
 615
 616         if (architecture == 'arm64') {
 617             assert (architecture == 'arm64') && (os == 'Ubuntu')
 618             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 619             def isBuild = (options != null) && (options['is_build_job'] == true)
 620             if (isFlow || isBuild) {
 621                 // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 622                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 623                 // Arm64 hardware.
 624                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 625             } else {
 626                 // Arm64 Linux test machines
 627                 if ((options != null) && (options['large_pages'] == true)) {
 628                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 629                 } else {
 630                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 631                 }
 632             }
 633         }
 634         else if (architecture == 'armem') {
 635             // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine,
 636             // using Docker.
 637             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 638         }
 639         else {
 640             // arm Ubuntu on hardware.
 641             assert (architecture == 'arm') && (os == 'Ubuntu')
 642             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 643             def isBuild = (options != null) && (options['is_build_job'] == true)
 644             if (isFlow || isBuild) {
 645                 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
 646                 // very important. Therefore, use latest or auto. Flow jobs don't need to use
 647                 // arm hardware.
 648                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 649             } else {
 650                 // arm Ubuntu test machine
 651                 // There is no tag (like, e.g., "arm-latest") for this, so don't call
 652                 // Utilities.setMachineAffinity. Just add the machine affinity
 653                 // manually. We specify the Helix queue name here.
 654                 job.with {
 655                     label('ubuntu.1404.arm32.open')
 656                 }
 657             }
 658         }
 659     }
 660 }
 661
 662 // setJobMachineAffinity: compute the machine affinity options for a job,
 663 // then set the job with those affinity options.
 664 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 665 {
 666     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 667            (!isBuildJob && isTestJob  && !isFlowJob) ||
 668            (!isBuildJob && !isTestJob && isFlowJob)
 669
 670     def affinityOptions = null
 671     def affinityArchitecture = architecture
 672
 673     if (os == "Windows_NT") {
 674         if (architecture in Constants.armWindowsCrossArchitectureList) {
 675             if (isBuildJob) {
 676                 affinityOptions = [ "use_arm64_build_machine" : true ]
 677             } else if (isTestJob) {
 678                 affinityOptions = [ "use_arm64_build_machine" : false ]
 679             } else if (isFlowJob) {
 680                 // For the flow jobs set the machine affinity as x64
 681                 affinityArchitecture = 'x64'
 682             }
 683         }
 684     }
 685     else {
 686         if (architecture == 'arm64') {
 687             if (isBuildJob) {
 688                 affinityOptions = ['is_build_job': true]
 689             } else if (isFlowJob) {
 690                 affinityOptions = ['is_flow_job': true]
 691             } else if (isTestJob) {
 692                 affinityOptions = [ "large_pages" : false ]
 693             }
 694         }
 695         else if (architecture == 'arm') {
 696             if (isBuildJob) {
 697                 affinityOptions = ['is_build_job': true]
 698             } else if (isFlowJob) {
 699                 affinityOptions = ['is_flow_job': true]
 700             }
 701         }
 702     }
 703
 704     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 705 }
 706
 707 def static isGCStressRelatedTesting(def scenario) {
 708     // The 'r2r_gcstress15' scenario is a basic scenario.
 709     // Detect it and make it a GCStress related.
 710     if (scenario == 'r2r_gcstress15')
 711     {
 712         return true;
 713     }
 714
 715     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 716     def scenarioName = scenario.toLowerCase()
 717     def isGCStressTesting = false
 718     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 719         if (k in gcStressTestEnvVars) {
 720             isGCStressTesting = true;
 721         }
 722     }
 723     return isGCStressTesting
 724 }
 725
 726 def static isCoreFxScenario(def scenario) {
 727     def corefx_prefix = 'corefx_'
 728     if (scenario.length() < corefx_prefix.length()) {
 729         return false
 730     }
 731     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 732 }
 733
 734 def static isR2RBaselineScenario(def scenario) {
 735     return (scenario == 'r2r')
 736 }
 737
 738 def static isR2RStressScenario(def scenario) {
 739     return Constants.r2rStressScenarios.containsKey(scenario)
 740 }
 741
 742 def static isR2RScenario(def scenario) {
 743     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 744 }
 745
 746 def static isJitStressScenario(def scenario) {
 747     return Constants.jitStressModeScenarios.containsKey(scenario)
 748 }
 749
 750 def static isLongGc(def scenario) {
 751     return (scenario == 'longgc' || scenario == 'gcsimulator')
 752 }
 753
 754 def static isJitDiff(def scenario) {
 755     return (scenario == 'jitdiff')
 756 }
 757
 758 def static isGcReliabilityFramework(def scenario) {
 759     return (scenario == 'gc_reliability_framework')
 760 }
 761
 762 def static isArmWindowsScenario(def scenario) {
 763     return Constants.validArmWindowsScenarios.containsKey(scenario)
 764 }
 765
 766 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 767     if (isBuildOnly == true) {
 768         os = 'Windows_NT_BuildOnly'
 769     }
 770
 771     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 772
 773     if (validOsPrTriggerArchConfigs == null) {
 774         return false
 775     }
 776
 777     if (validOsPrTriggerArchConfigs[architecture] != null) {
 778         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 779
 780         if (!(configuration in validOsPrTriggerConfigs)) {
 781             return false
 782         }
 783     } else {
 784         return false
 785     }
 786
 787     return true
 788 }
 789
 790 def static getFxBranch(def branch) {
 791     def fxBranch = branch
 792     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 793     // branch even though CoreFX doesn't have such a branch.
 794     if (branch == 'dev/unix_test_workflow') {
 795         fxBranch = 'master'
 796     }
 797     return fxBranch
 798 }
 799
 800 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 801     // 2 hours (120 minutes) is the default timeout
 802     def timeout = 120
 803     def innerLoop = (scenario == "innerloop")
 804
 805     if (!innerLoop) {
 806         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 807         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 808         timeout = 240
 809     }
 810
 811     if (!isBuildOnly) {
 812         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 813         if (isGCStressRelatedTesting(scenario)) {
 814             timeout = 4320
 815         }
 816         else if (isCoreFxScenario(scenario)) {
 817             timeout = 360
 818         }
 819         else if (isJitStressScenario(scenario)) {
 820             timeout = 300
 821         }
 822         else if (isR2RBaselineScenario(scenario)) {
 823             timeout = 240
 824         }
 825         else if (isLongGc(scenario)) {
 826             timeout = 1440
 827         }
 828         else if (isJitDiff(scenario)) {
 829             timeout = 240
 830         }
 831         else if (isGcReliabilityFramework(scenario)) {
 832             timeout = 1440
 833         }
 834         else if (architecture == 'armem' || architecture == 'arm64') {
 835             timeout = 240
 836         }
 837
 838         if (architecture == 'arm') {
 839             // ARM32 machines are particularly slow.
 840             timeout += 120
 841         }
 842     }
 843
 844     if (configuration == 'Debug') {
 845         // Debug runs can be very slow. Add an hour.
 846         timeout += 60
 847     }
 848
 849     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 850         // AltJit runs compile all methods twice.
 851         timeout *= 2
 852     }
 853
 854     // If we've changed the timeout from the default, set it in the job.
 855
 856     if (timeout != 120) {
 857         Utilities.setJobTimeout(newJob, timeout)
 858     }
 859 }
 860
 861 def static getJobFolder(def scenario) {
 862     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 863         return 'jitstress'
 864     }
 865     if (scenario == 'illink') {
 866         return 'illink'
 867     }
 868     return ''
 869 }
 870
 871 def static getStressModeDisplayName(def scenario) {
 872     def displayStr = ''
 873     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 874         def prefixLength = 'COMPlus_'.length()
 875         if (k.length() >= prefixLength) {
 876             def modeName = k.substring(prefixLength, k.length())
 877             displayStr += ' ' + modeName + '=' + v
 878         }
 879     }
 880
 881     if (isCoreFxScenario(scenario)) {
 882         displayStr = ('CoreFx ' + displayStr).trim()
 883     }
 884
 885     return displayStr
 886 }
 887
 888 def static getR2RDisplayName(def scenario) {
 889     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 890     def displayStr = scenario
 891     def prefixLength = 'r2r_'.length()
 892     if (displayStr.length() >= prefixLength) {
 893         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 894     } else if (scenario == 'r2r') {
 895         displayStr = "R2R"
 896     }
 897     return displayStr
 898 }
 899
 900 //
 901 // Functions to create an environment script.
 902 //      envScriptCreate -- initialize the script (call first)
 903 //      envScriptFinalize -- finalize the script (call last)
 904 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 905 //      envScriptAppendExistingScript -- append an existing script to the generated script
 906 //
 907 // Each script returns a string of commands. Concatenate all the strings together before
 908 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 909 //
 910
 911 // Initialize the environment setting script.
 912 def static envScriptCreate(def os, def stepScriptLocation) {
 913     def stepScript = ''
 914     if (os == 'Windows_NT') {
 915         stepScript += "echo Creating TestEnv script\r\n"
 916         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 917
 918         // Create at least an empty script.
 919         stepScript += "echo. > ${stepScriptLocation}\r\n"
 920     }
 921     else {
 922         stepScript += "echo Creating environment setting script\n"
 923         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 924     }
 925
 926     return stepScript
 927 }
 928
 929 // Generates the string for setting stress mode variables.
 930 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 931     def stepScript = ''
 932     if (os == 'Windows_NT') {
 933         stressModeVars.each{ k, v ->
 934             // Write out what we are writing to the script file
 935             stepScript += "echo Setting ${k}=${v}\r\n"
 936             // Write out the set itself to the script file`
 937             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 938         }
 939     }
 940     else {
 941         stressModeVars.each{ k, v ->
 942             // Write out what we are writing to the script file
 943             stepScript += "echo Setting ${k}=${v}\n"
 944             // Write out the set itself to the script file`
 945             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 946         }
 947     }
 948
 949     return stepScript
 950 }
 951
 952 // Append an existing script to an environment script.
 953 // Returns string of commands to do this.
 954 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 955     assert (os == 'Windows_NT')
 956     def stepScript = ''
 957
 958     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 959     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 960
 961     return stepScript
 962 }
 963
 964 // Finalize an environment setting script.
 965 // Returns string of commands to do this.
 966 def static envScriptFinalize(def os, def stepScriptLocation) {
 967     def stepScript = ''
 968
 969     if (os == 'Windows_NT') {
 970         // Display the resulting script. This is useful when looking at the output log file.
 971         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 972         stepScript += "type ${stepScriptLocation}\r\n"
 973     }
 974     else {
 975         stepScript += "chmod +x ${stepScriptLocation}\n"
 976     }
 977
 978     return stepScript
 979 }
 980
 981 def static isNeedDocker(def architecture, def os, def isBuild) {
 982     if (isBuild) {
 983         if (architecture == 'x86' && os == 'Ubuntu') {
 984             return true
 985         }
 986         else if (architecture == 'armem') {
 987             return true
 988         }
 989         else if (architecture == 'arm') {
 990             if (os == 'Ubuntu') {
 991                 return true
 992             }
 993         }
 994         else if (architecture == 'arm64') {
 995             if (os == 'Ubuntu') {
 996                 return true
 997             }
 998         }
 999     }
1000     else {
1001         if (architecture == 'x86' && os == 'Ubuntu') {
1002             return true
1003         }
1004     }
1005     return false
1006 }
1007
1008 def static getDockerImageName(def architecture, def os, def isBuild) {
1009     // We must change some docker private images to official later
1010     if (isBuild) {
1011         if (architecture == 'x86' && os == 'Ubuntu') {
1012             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1013         }
1014         else if (architecture == 'armem') {
1015             if (os == 'Ubuntu') {
1016                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180405193556"
1017             }
1018             else if (os == 'Ubuntu16.04') {
1019                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180404203310"
1020             }
1021             else if (os == 'Tizen') {
1022                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1023             }
1024         }
1025         else if (architecture == 'arm') {
1026             if (os == 'Ubuntu') {
1027                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1028             }
1029         }
1030         else if (architecture == 'arm64') {
1031             if (os == 'Ubuntu') {
1032                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1033             }
1034         }
1035     }
1036     else {
1037         if (architecture == 'x86' && os == 'Ubuntu') {
1038             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1039         }
1040     }
1041     println("Unknown architecture to use docker: ${architecture} ${os}");
1042     assert false
1043 }
1044
1045
1046 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1047 // and only allowing using this hardware in some specific branches.
1048 def static jobRequiresLimitedHardware(def architecture, def os) {
1049     if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
1050         // These test jobs require ARM64 hardware
1051         return true
1052     }
1053     else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1054         // These test jobs require Linux/arm32 hardware
1055         return true
1056     }
1057     else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
1058         // These test jobs require Linux/arm64 hardware
1059         return true
1060     }
1061     else {
1062         return false
1063     }
1064 }
1065
1066 // Calculates the name of the build job based on some typical parameters.
1067 //
1068 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1069     // If the architecture is x64, do not add that info into the build name.
1070     // Need to change around some systems and other builds to pick up the right builds
1071     // to do that.
1072
1073     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1074     if (isBuildOnly) {
1075         suffix += '_bld'
1076     }
1077     def baseName = ''
1078     switch (architecture) {
1079         case 'x64':
1080             if (scenario == 'normal') {
1081                 // For now we leave x64 off of the name for compatibility with other jobs
1082                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1083             }
1084             else if (scenario == 'formatting') {
1085                 // we don't care about the configuration for the formatting job. It runs all configs
1086                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1087             }
1088             else {
1089                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1090             }
1091             break
1092         case 'arm64':
1093             if (os.toLowerCase() == "windows_nt") {
1094                 // These are cross builds
1095                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1096             }
1097             else {
1098                 // Defaults to a small page size set of machines.
1099                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1100             }
1101             break
1102         case 'armem':
1103             // These are cross builds
1104             if (os == 'Tizen') {
1105                 // ABI: softfp
1106                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1107             }
1108             else {
1109                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1110             }
1111             break
1112         case 'arm':
1113             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1114             break
1115         case 'x86':
1116         case 'x86_arm_altjit':
1117         case 'x64_arm64_altjit':
1118             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1119             break
1120         default:
1121             println("Unknown architecture: ${architecture}");
1122             assert false
1123             break
1124     }
1125
1126     return baseName + suffix
1127 }
1128
1129 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1130
1131     // Limited hardware is restricted for non-PR triggers to certain branches.
1132     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1133         return
1134     }
1135
1136     // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
1137     if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1138          return
1139     }
1140
1141     // Check scenario.
1142     switch (scenario) {
1143         case 'innerloop':
1144             break
1145         case 'normal':
1146             switch (architecture) {
1147                 case 'x64':
1148                 case 'x86':
1149                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1150                         addPeriodicTriggerHelper(job, '@daily')
1151                     }
1152                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1153                         addGithubPushTriggerHelper(job)
1154                     }
1155                     break
1156                 case 'arm64':
1157                     if (os == 'Windows_NT') {
1158                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1159                         if (isFlowJob) {
1160                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1161                             // Do the builds daily.
1162                             addPeriodicTriggerHelper(job, '@daily')
1163                         }
1164                     }
1165                     else {
1166                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1167                         if (isFlowJob) {
1168                             addPeriodicTriggerHelper(job, '@daily')
1169                         }
1170                     }
1171                     break
1172                 case 'arm':
1173                     if (os == 'Windows_NT') {
1174                         // Only the flow jobs get triggers; the build and test jobs are triggered by the flow job.
1175                         if (isFlowJob) {
1176                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1177                             // Do the builds daily.
1178                             addPeriodicTriggerHelper(job, '@daily')
1179                         }
1180                     }
1181                     else {
1182                         assert os == 'Ubuntu'
1183                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1184                         if (isFlowJob) {
1185                             // Currently no push triggers, with limited arm Linux hardware.
1186                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1187                             addPeriodicTriggerHelper(job, '@daily')
1188                         }
1189                     }
1190                     break
1191                 case 'armem':
1192                     addGithubPushTriggerHelper(job)
1193                     break
1194                 case 'x86_arm_altjit':
1195                 case 'x64_arm64_altjit':
1196                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1197                     if (configuration == 'Checked') {
1198                         addGithubPushTriggerHelper(job)
1199                     }
1200                     break
1201                 default:
1202                     println("Unknown architecture: ${architecture}");
1203                     assert false
1204                     break
1205             }
1206             break
1207         case 'r2r':
1208             assert !(os in bidailyCrossList)
1209             // r2r gets a push trigger for checked/release
1210             if (configuration == 'Checked' || configuration == 'Release') {
1211                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1212                 if (architecture == 'x64' && os != 'OSX10.12') {
1213                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1214                     if (isFlowJob || os == 'Windows_NT') {
1215                         addGithubPushTriggerHelper(job)
1216                     }
1217                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1218                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1219                     if (isFlowJob) {
1220                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1221                     }
1222                 }
1223                 // For x86, only add per-commit jobs for Windows
1224                 else if (architecture == 'x86') {
1225                     if (os == 'Windows_NT') {
1226                         addGithubPushTriggerHelper(job)
1227                     }
1228                 }
1229                 // arm64 r2r jobs should only run weekly.
1230                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1231                 else if (architecture == 'arm64') {
1232                     if (os == 'Windows_NT') {
1233                         if (isFlowJob) {
1234                             addPeriodicTriggerHelper(job, '@weekly')
1235                         }
1236                     }
1237                 }
1238                 // arm r2r jobs should only run weekly.
1239                 else if (architecture == 'arm') {
1240                     if (isFlowJob) {
1241                         addPeriodicTriggerHelper(job, '@weekly')
1242                     }
1243                 }
1244             }
1245             break
1246         case 'r2r_jitstress1':
1247         case 'r2r_jitstress2':
1248         case 'r2r_jitstressregs1':
1249         case 'r2r_jitstressregs2':
1250         case 'r2r_jitstressregs3':
1251         case 'r2r_jitstressregs4':
1252         case 'r2r_jitstressregs8':
1253         case 'r2r_jitstressregs0x10':
1254         case 'r2r_jitstressregs0x80':
1255         case 'r2r_jitstressregs0x1000':
1256         case 'r2r_jitminopts':
1257         case 'r2r_jitforcerelocs':
1258         case 'r2r_gcstress15':
1259             assert !(os in bidailyCrossList)
1260
1261             // GCStress=C is currently not supported on OS X
1262             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1263                 break
1264             }
1265
1266             // GC Stress 15 r2r gets a push trigger for checked/release
1267             if (configuration == 'Checked' || configuration == 'Release') {
1268                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1269                 if (architecture == 'x64') {
1270                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1271                     if (isFlowJob || os == 'Windows_NT') {
1272                         // Add a weekly periodic trigger
1273                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1274                     }
1275                 }
1276                 // For x86, only add periodic jobs for Windows
1277                 else if (architecture == 'x86') {
1278                     if (os == 'Windows_NT') {
1279                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1280                     }
1281                 }
1282                 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1283                 else if (architecture == 'arm64') {
1284                     if (os == 'Windows_NT') {
1285                         if (isFlowJob) {
1286                             addPeriodicTriggerHelper(job, '@weekly')
1287                         }
1288                     }
1289                 }
1290                 else if (architecture == 'arm') {
1291                     if (isFlowJob) {
1292                         addPeriodicTriggerHelper(job, '@weekly')
1293                     }
1294                 }
1295             }
1296             break
1297         case 'longgc':
1298             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1299             assert configuration == 'Release'
1300             assert architecture == 'x64'
1301             addPeriodicTriggerHelper(job, '@daily')
1302             // TODO: Add once external email sending is available again
1303             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1304             break
1305         case 'gcsimulator':
1306             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1307             assert configuration == 'Release'
1308             assert architecture == 'x64'
1309             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1310             // TODO: Add once external email sending is available again
1311             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1312             break
1313         case 'standalone_gc':
1314             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1315             assert (configuration == 'Release' || configuration == 'Checked')
1316             // TODO: Add once external email sending is available again
1317             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1318             addPeriodicTriggerHelper(job, '@daily')
1319             break
1320         case 'gc_reliability_framework':
1321             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1322             assert (configuration == 'Release' || configuration == 'Checked')
1323             // Only triggered by phrase.
1324             break
1325         case 'ilrt':
1326             assert !(os in bidailyCrossList)
1327             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1328             if (architecture == 'x64' && configuration == 'Release') {
1329                 // We don't expect to see a job generated except in these scenarios
1330                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1331                 if (isFlowJob || os == 'Windows_NT') {
1332                     addPeriodicTriggerHelper(job, '@daily')
1333                 }
1334             }
1335             break
1336         case 'jitdiff':
1337             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1338             assert configuration == 'Checked'
1339             assert (architecture == 'x64' || architecture == 'x86')
1340             addGithubPushTriggerHelper(job)
1341             break
1342         case 'formatting':
1343             assert (os == 'Windows_NT' || os == "Ubuntu")
1344             assert architecture == 'x64'
1345             addGithubPushTriggerHelper(job)
1346             break
1347         case 'jitstressregs1':
1348         case 'jitstressregs2':
1349         case 'jitstressregs3':
1350         case 'jitstressregs4':
1351         case 'jitstressregs8':
1352         case 'jitstressregs0x10':
1353         case 'jitstressregs0x80':
1354         case 'jitstressregs0x1000':
1355         case 'minopts':
1356         case 'forcerelocs':
1357         case 'jitstress1':
1358         case 'jitstress2':
1359         case 'jitstress2_jitstressregs1':
1360         case 'jitstress2_jitstressregs2':
1361         case 'jitstress2_jitstressregs3':
1362         case 'jitstress2_jitstressregs4':
1363         case 'jitstress2_jitstressregs8':
1364         case 'jitstress2_jitstressregs0x10':
1365         case 'jitstress2_jitstressregs0x80':
1366         case 'jitstress2_jitstressregs0x1000':
1367         case 'tailcallstress':
1368         case 'jitsse2only':
1369         case 'jitnosimd':
1370         case 'jitnox86hwintrinsic':
1371         case 'jitincompletehwintrinsic':
1372         case 'jitx86hwintrinsicnoavx':
1373         case 'jitx86hwintrinsicnoavx2':
1374         case 'jitx86hwintrinsicnosimd':
1375         case 'corefx_baseline':
1376         case 'corefx_minopts':
1377         case 'corefx_jitstress1':
1378         case 'corefx_jitstress2':
1379         case 'corefx_jitstressregs1':
1380         case 'corefx_jitstressregs2':
1381         case 'corefx_jitstressregs3':
1382         case 'corefx_jitstressregs4':
1383         case 'corefx_jitstressregs8':
1384         case 'corefx_jitstressregs0x10':
1385         case 'corefx_jitstressregs0x80':
1386         case 'corefx_jitstressregs0x1000':
1387         case 'zapdisable':
1388             if (os == 'CentOS7.1') {
1389                 break
1390             }
1391             if (os in bidailyCrossList) {
1392                 break
1393             }
1394             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1395             // build for the flow job. We don't need cron jobs for these. Note that the
1396             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1397             // function very early, so only non-Windows gets here.
1398             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1399                 break
1400             }
1401             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1402                 break
1403             }
1404             assert (os == 'Windows_NT') || (os in Constants.crossList)
1405             if (jobRequiresLimitedHardware(architecture, os)) {
1406                 addPeriodicTriggerHelper(job, '@weekly')
1407             }
1408             else {
1409                 addPeriodicTriggerHelper(job, '@daily')
1410             }
1411             break
1412         case 'heapverify1':
1413         case 'gcstress0x3':
1414             if (os == 'CentOS7.1') {
1415                 break
1416             }
1417             if (os in bidailyCrossList) {
1418                 break
1419             }
1420             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1421                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1422                 break
1423             }
1424             assert (os == 'Windows_NT') || (os in Constants.crossList)
1425             addPeriodicTriggerHelper(job, '@weekly')
1426             break
1427         case 'gcstress0xc':
1428         case 'gcstress0xc_zapdisable':
1429         case 'gcstress0xc_zapdisable_jitstress2':
1430         case 'gcstress0xc_zapdisable_heapverify1':
1431         case 'gcstress0xc_jitstress1':
1432         case 'gcstress0xc_jitstress2':
1433         case 'gcstress0xc_minopts_heapverify1':
1434             if (os == 'CentOS7.1') {
1435                 break
1436             }
1437             if (os == 'OSX10.12') {
1438                 // GCStress=C is currently not supported on OS X
1439                 break
1440             }
1441             if (os in bidailyCrossList) {
1442                 break
1443             }
1444             if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1445                 // TODO: should we have cron jobs for arm64 Linux GCStress?
1446                 break
1447             }
1448             assert (os == 'Windows_NT') || (os in Constants.crossList)
1449             addPeriodicTriggerHelper(job, '@weekly')
1450             break
1451
1452         case 'illink':
1453             // Testing on other operating systems TBD
1454             assert (os == 'Windows_NT' || os == 'Ubuntu')
1455             if (architecture == 'x64' || architecture == 'x86') {
1456                 if (configuration == 'Checked') {
1457                     addPeriodicTriggerHelper(job, '@daily')
1458                 }
1459             }
1460             break
1461
1462         case 'tieredcompilation':
1463         case 'corefx_tieredcompilation':
1464             // No periodic jobs just yet, still testing
1465             break
1466
1467         default:
1468             println("Unknown scenario: ${scenario}");
1469             assert false
1470             break
1471     }
1472     return
1473 }
1474
1475 // **************************
1476 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1477 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1478 // builds will do a couple extra steps.
1479 // **************************
1480
1481 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1482 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1483 // it's just used for internal builds
1484 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1485 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1486     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1487
1488     if (isWindowsBuildOnlyJob) {
1489         return
1490     }
1491
1492     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1493     // Non pull request builds.
1494     if (!isPR) {
1495         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1496         return
1497     }
1498
1499      def arm64Users = [
1500         'adityamandaleeka',
1501         'AndyAyersMS',
1502         'briansull',
1503         'BruceForstall',
1504         'CarolEidt',
1505         'cmckinsey',
1506         'echesakovMSFT',
1507         'erozenfeld',
1508         'janvorli',
1509         'jashook',
1510         'JosephTremoulet',
1511         'pgodeq',
1512         'russellhadley',
1513         'RussKeldorph',
1514         'sandreenko',
1515         'sdmaclea',
1516         'swaroop-sridhar',
1517         'jkotas',
1518         'markwilkie',
1519         'weshaggard'
1520     ]
1521
1522     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1523     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1524     def osGroup = getOSGroup(os)
1525     switch (architecture) {
1526         case 'x64': // editor brace matching: {
1527             if (scenario == 'formatting') {
1528                 assert configuration == 'Checked'
1529                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1530                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
1531                 }
1532
1533                 break
1534             }
1535
1536             switch (os) {
1537                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1538                 case 'Debian8.4':
1539                 case 'RHEL7.2':
1540                     if (scenario == 'innerloop') {
1541                         assert !isFlowJob
1542                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1543                     }
1544                     else if (scenario == 'normal') {
1545                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1546                     }
1547                     break
1548
1549                 case 'Ubuntu16.04':
1550                     assert !isFlowJob
1551                     assert scenario != 'innerloop'
1552                     // Distinguish with the other architectures (arm and x86)
1553                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1554                     break
1555
1556                 case 'Fedora24':
1557                 case 'Ubuntu16.10':
1558                     assert !isFlowJob
1559                     assert scenario != 'innerloop'
1560                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1561                     break
1562
1563                 case 'Ubuntu':
1564                     if (scenario == 'illink') {
1565                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1566                         break
1567                     }
1568                     // fall through
1569
1570                 case 'OSX10.12':
1571                     // Triggers on the non-flow jobs aren't necessary here
1572                     // Corefx testing uses non-flow jobs.
1573                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1574                         break
1575                     }
1576                     switch (scenario) {
1577                         case 'innerloop':
1578                             // PR Triggered jobs. These jobs will run pri0 tests.
1579                             if (configuration == 'Checked') {
1580                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1581                             }
1582                             break
1583
1584                         case 'normal':
1585                             // OSX uses checked for default PR tests
1586                             if (configuration == 'Checked') {
1587                                 // Default trigger
1588                                 assert !job.name.contains("centos")
1589                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1590                             }
1591                             break
1592
1593                         case 'jitdiff':
1594                             if (configuration == 'Checked') {
1595                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1596                             }
1597                             break
1598
1599                         case 'ilrt':
1600                             if (configuration == 'Release') {
1601                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1602                             }
1603                             break
1604
1605                         case 'longgc':
1606                             if (configuration == 'Release') {
1607                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1608                             }
1609                             break
1610
1611                         case 'gcsimulator':
1612                             if (configuration == 'Release') {
1613                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1614                             }
1615                             break
1616
1617                         case 'standalone_gc':
1618                             if (configuration == 'Release' || configuration == 'Checked') {
1619                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1620                             }
1621                             break
1622
1623                         case 'gc_reliability_framework':
1624                             if (configuration == 'Release' || configuration == 'Checked') {
1625                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1626                             }
1627                             break
1628
1629                         default:
1630                             if (isJitStressScenario(scenario)) {
1631                                 def displayStr = getStressModeDisplayName(scenario)
1632                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1633                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1634                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1635                             }
1636                             else if (isR2RScenario(scenario)) {
1637                                 if (configuration == 'Release' || configuration == 'Checked') {
1638                                     def displayStr = getR2RDisplayName(scenario)
1639                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1640                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1641                                 }
1642                             }
1643                             else {
1644                                 println("Unknown scenario: ${scenario}");
1645                                 assert false
1646                             }
1647                             break
1648
1649                     }
1650                     break
1651
1652                 case 'CentOS7.1':
1653                     switch (scenario) {
1654                         case 'innerloop':
1655                             // CentOS uses checked for default PR tests while debug is build only
1656                             if (configuration == 'Debug') {
1657                                 // Default trigger
1658                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1659                             }
1660
1661                             // Make sure this is a flow job to get build and test.
1662                             if (configuration == 'Checked' && isFlowJob) {
1663                                 assert job.name.contains("flow")
1664                                 // Default trigger
1665                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1666                             }
1667                             break
1668
1669                         case 'normal':
1670                             // Make sure this is a flow job to get build and test.
1671                             if (configuration == 'Checked' && isFlowJob) {
1672                                 assert job.name.contains("flow")
1673                                 // Default trigger
1674                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1675                             }
1676                             break
1677
1678                         default:
1679                             if (isR2RScenario(scenario)) {
1680                                 if (configuration == 'Release' || configuration == 'Checked') {
1681                                     def displayStr = getR2RDisplayName(scenario)
1682                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1683                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1684                                 }
1685                             }
1686                             break
1687
1688                     }
1689                     break
1690
1691                 case 'Windows_NT':
1692                     switch (scenario) {
1693                         case 'innerloop':
1694                             // Default trigger
1695                             if (configuration == 'Checked' || configuration == 'Release') {
1696                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1697                             }
1698                             break
1699
1700                         case 'normal':
1701                             if (configuration == 'Checked') {
1702                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+Build and Test.*")
1703                             }
1704                             break
1705
1706                         case 'jitdiff':
1707                             if (configuration == 'Checked') {
1708                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1709                             }
1710                             break
1711
1712                         case 'ilrt':
1713                             if (configuration == 'Release') {
1714                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1715                             }
1716                             break
1717
1718                         case 'longgc':
1719                             if (configuration == 'Release') {
1720                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1721                             }
1722                             break
1723
1724                         case 'gcsimulator':
1725                             if (configuration == 'Release') {
1726                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1727                             }
1728                             break
1729
1730                         case 'standalone_gc':
1731                             if (configuration == 'Release' || configuration == 'Checked') {
1732                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1733                             }
1734                             break
1735
1736                         case 'gc_reliability_framework':
1737                             if (configuration == 'Release' || configuration == 'Checked') {
1738                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1739                             }
1740                             break
1741
1742                         case 'illink':
1743                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1744                             break
1745                         case 'corefx_innerloop':
1746                             if (configuration == 'Release' || configuration == 'Checked') {
1747                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests")
1748                             }
1749                             break
1750
1751                         default:
1752                             if (isJitStressScenario(scenario)) {
1753                                 def displayStr = getStressModeDisplayName(scenario)
1754                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1755                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1756                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1757                             }
1758                             else if (isR2RScenario(scenario)) {
1759                                 if (configuration == 'Release' || configuration == 'Checked') {
1760                                     def displayStr = getR2RDisplayName(scenario)
1761                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1762                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1763                                 }
1764                             }
1765                             else {
1766                                 println("Unknown scenario: ${scenario}");
1767                                 assert false
1768                             }
1769                             break
1770
1771                     }
1772                     break
1773
1774                 default:
1775                     println("Unknown os: ${os}");
1776                     assert false
1777                     break
1778
1779             }
1780
1781             break
1782         // editor brace matching: }
1783
1784         case 'armem': // editor brace matching: {
1785             job.with {
1786                 publishers {
1787                     azureVMAgentPostBuildAction {
1788                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1789                     }
1790                 }
1791             }
1792
1793             switch (os) {
1794                 case 'Ubuntu':
1795                 case 'Ubuntu16.04':
1796                     assert scenario != 'innerloop'
1797                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1798                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1799                     break
1800
1801                 case 'Tizen':
1802                     architecture = 'armel'
1803
1804                     if (scenario == 'innerloop') {
1805                         if (configuration == 'Checked') {
1806                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1807                         }
1808                     }
1809                     else {
1810                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1811                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1812                     }
1813                     break
1814             }
1815
1816             break
1817         // editor brace matching: }
1818
1819         case 'arm': // editor brace matching: {
1820
1821             // Triggers on the non-flow jobs aren't necessary
1822             if (!isFlowJob) {
1823                 break
1824             }
1825
1826             // Set up a private trigger
1827             def contextString = "${os} ${architecture} Cross ${configuration}"
1828             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1829             if (scenario == 'innerloop') {
1830                 contextString += " Innerloop"
1831                 triggerString += "\\W+Innerloop"
1832             }
1833             else {
1834                 contextString += " ${scenario}"
1835                 triggerString += "\\W+${scenario}"
1836             }
1837
1838             if (configuration == 'Debug') {
1839                 contextString += " Build"
1840                 triggerString += "\\W+Build"
1841             } else {
1842                 contextString += " Build and Test"
1843                 triggerString += "\\W+Build and Test"
1844             }
1845
1846             triggerString += ".*"
1847
1848             switch (os) {
1849                 case 'Ubuntu':
1850                     if (scenario == 'innerloop') {
1851                         if (configuration == 'Checked') {
1852                             Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
1853                         }
1854                     }
1855                     else {
1856                         Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
1857                     }
1858                     break
1859
1860                 case 'Windows_NT':
1861                     switch (scenario) {
1862                         case 'innerloop':
1863                             // Only Checked is an innerloop trigger.
1864                             if (configuration == 'Checked')
1865                             {
1866                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1867                             }
1868                             break
1869                         case 'normal':
1870                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1871                             break
1872                         default:
1873                             // Stress jobs will use this code path.
1874                             if (isArmWindowsScenario(scenario)) {
1875                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1876                             }
1877                             break
1878                     }
1879                     break
1880                 default:
1881                     println("NYI os: ${os}");
1882                     assert false
1883                     break
1884             }
1885             break
1886         // editor brace matching: }
1887         case 'arm64': // editor brace matching: {
1888             // Set up a private trigger
1889             def contextString = "${os} ${architecture} Cross ${configuration}"
1890             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1891
1892             if (scenario == 'innerloop') {
1893                 contextString += " Innerloop"
1894                 triggerString += "\\W+Innerloop"
1895             }
1896             else {
1897                 contextString += " ${scenario}"
1898                 triggerString += "\\W+${scenario}"
1899             }
1900
1901             if (configuration == 'Debug') {
1902                 contextString += " Build"
1903                 triggerString += "\\W+Build"
1904             } else {
1905                 contextString += " Build and Test"
1906                 triggerString += "\\W+Build and Test"
1907             }
1908
1909             triggerString += ".*"
1910
1911             switch (os) {
1912                 case 'Ubuntu':
1913                 case 'Ubuntu16.04':
1914                     switch (scenario) {
1915                         case 'innerloop':
1916                             if (configuration == 'Debug' && !isFlowJob) {
1917                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1918                             }
1919
1920                             break
1921                         case 'normal':
1922                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1923                             break
1924                         default:
1925                             if (isR2RScenario(scenario)) {
1926                                 if (configuration == 'Checked' || configuration == 'Release') {
1927                                     def displayStr = getR2RDisplayName(scenario)
1928                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1929                                 }
1930                             }
1931                             break
1932                     }
1933                     break
1934
1935                 case 'Windows_NT':
1936                     // Triggers on the non-flow jobs aren't necessary here
1937                     if (!isFlowJob) {
1938                         break
1939                     }
1940
1941                     assert isArmWindowsScenario(scenario)
1942                     switch (scenario) {
1943                         case 'innerloop':
1944                             if (configuration == 'Checked') {
1945                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1946                             }
1947
1948                             break
1949                         case 'normal':
1950                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1951                             break
1952                         default:
1953                             // Stress jobs will use this code path.
1954                             if (isArmWindowsScenario(scenario)) {
1955                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1956                             }
1957                             break
1958                     }
1959                     break
1960                 default:
1961                     println("NYI os: ${os}");
1962                     assert false
1963                     break
1964             }
1965             break
1966
1967         // editor brace matching: }
1968         case 'x86': // editor brace matching: {
1969             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1970             if (os == 'Ubuntu') {
1971                 // Triggers on the non-flow jobs aren't necessary here
1972                 if (!isFlowJob) {
1973                     break
1974                 }
1975
1976                 // on-demand only for ubuntu x86
1977                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1978                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1979                 break
1980
1981             }
1982             switch (scenario) {
1983                 case 'innerloop':
1984                     if (configuration == 'Checked' || configuration == 'Release') {
1985                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1986                     }
1987                     break
1988
1989                 case 'normal':
1990                     if (configuration == 'Checked') {
1991                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1992                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1993                     }
1994                     break
1995
1996                 case 'ilrt':
1997                     if (configuration == 'Release') {
1998                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1999                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2000                     }
2001                     break
2002
2003                 case 'longgc':
2004                     if (configuration == 'Release') {
2005                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
2006                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2007                     }
2008                     break
2009
2010                 case 'gcsimulator':
2011                     if (configuration == 'Release') {
2012                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
2013                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2014                     }
2015                     break
2016
2017                 case 'standalone_gc':
2018                     if (configuration == 'Release' || configuration == 'Checked') {
2019                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
2020                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2021                     }
2022                     break
2023
2024                 case 'illink':
2025                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2026                     break
2027
2028                 default:
2029                     if (isJitStressScenario(scenario)) {
2030                         def displayStr = getStressModeDisplayName(scenario)
2031                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
2032                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2033                     }
2034                     else if (isR2RScenario(scenario)) {
2035                         if (configuration == 'Release' || configuration == 'Checked') {
2036                             def displayStr = getR2RDisplayName(scenario)
2037                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
2038                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2039                         }
2040                     }
2041                     else {
2042                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
2043                         assert false
2044                     }
2045                     break
2046
2047             }
2048             break
2049
2050          // editor brace matching: }
2051         case 'x64_arm64_altjit':
2052         case 'x86_arm_altjit': // editor brace matching: {
2053             assert (os == 'Windows_NT')
2054             switch (scenario) {
2055                 case 'normal':
2056                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
2057                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
2058                     break
2059                 default:
2060                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
2061                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
2062                     break
2063             }
2064             break
2065
2066         // editor brace matching: }
2067         default:
2068             println("Unknown architecture: ${architecture}");
2069             assert false
2070             break
2071     }
2072 }
2073
2074 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2075     def buildCommands = []
2076     def osGroup = getOSGroup(os)
2077     def lowerConfiguration = configuration.toLowerCase()
2078
2079     def priority = '1'
2080     if (scenario == 'innerloop') {
2081         priority = '0'
2082     }
2083
2084     def doCoreFxTesting = isCoreFxScenario(scenario)
2085
2086     // Calculate the build steps, archival, and xunit results
2087     switch (os) {
2088         case 'Windows_NT': // editor brace matching: {
2089             switch (architecture) {
2090                 case 'x64':
2091                 case 'x86':
2092                 case 'x86_arm_altjit':
2093                 case 'x64_arm64_altjit':
2094                     def arch = architecture
2095                     def buildOpts = ''
2096                     if (architecture == 'x86_arm_altjit') {
2097                         arch = 'x86'
2098                     }
2099                     else if (architecture == 'x64_arm64_altjit') {
2100                         arch = 'x64'
2101                     }
2102
2103                     if (scenario == 'formatting') {
2104                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2105                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2106                         break
2107                     }
2108
2109                     if (scenario == 'illink') {
2110                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2111                     }
2112
2113                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2114                     if ((lowerConfiguration == 'release') &&
2115                         (scenario in Constants.basicScenarios) &&
2116                         (architecture != 'x86_arm_altjit') &&
2117                         (architecture != 'x64_arm64_altjit')) {
2118
2119                         buildOpts += ' -enforcepgo'
2120                     }
2121
2122                     if (doCoreFxTesting) {
2123                         buildOpts += ' skiptests';
2124                     } else {
2125                         buildOpts += " -priority=${priority}"
2126                     }
2127
2128                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2129                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2130                     // 35 characters long.
2131
2132                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2133
2134                     if (!isBuildOnly) {
2135                         def runtestArguments = ''
2136                         def testOpts = 'collectdumps'
2137
2138                         if (isR2RScenario(scenario)) {
2139
2140                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2141                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2142                             // to cause the tests to be crossgen'ed.
2143
2144                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2145                                 testOpts += ' crossgenaltjit protononjit.dll'
2146                             } else {
2147                                 testOpts += ' crossgen'
2148                             }
2149
2150                             testOpts += ' runcrossgentests'
2151                         }
2152                         else if (scenario == 'jitdiff') {
2153                             testOpts += ' jitdisasm crossgen'
2154                         }
2155                         else if (scenario == 'ilrt') {
2156                             testOpts += ' ilasmroundtrip'
2157                         }
2158                         else if (isLongGc(scenario)) {
2159                             testOpts += " ${scenario} sequential"
2160                         }
2161                         else if (scenario == 'standalone_gc') {
2162                             testOpts += ' gcname clrgc.dll'
2163                         }
2164                         else if (scenario == 'illink') {
2165                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2166                         }
2167
2168                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2169                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2170                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2171                         // which is read by the xunit harness.)
2172                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2173                         {
2174                             def timeout = 1800000
2175                             testOpts += " timeout ${timeout}"
2176                         }
2177
2178                         // If we are running a stress mode, we should write out the set of key
2179                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2180
2181                         def envScriptPath = ''
2182                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2183                             def buildCommandsStr = ''
2184                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2185                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2186
2187                             if (isJitStressScenario(scenario)) {
2188                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2189                             }
2190                             else if (isR2RStressScenario(scenario)) {
2191                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2192                             }
2193
2194                             if (architecture == 'x86_arm_altjit') {
2195                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2196                             }
2197                             else if (architecture == 'x64_arm64_altjit') {
2198                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2199                             }
2200
2201                             envScriptFinalize(os, envScriptPath)
2202
2203                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2204                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2205                             // into a single script.
2206                             buildCommands += buildCommandsStr
2207                         }
2208                         else if (architecture == 'x86_arm_altjit') {
2209                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2210                         }
2211                         else if (architecture == 'x64_arm64_altjit') {
2212                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2213                         }
2214                         if (envScriptPath != '') {
2215                             testOpts += " TestEnv ${envScriptPath}"
2216                         }
2217
2218                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2219
2220                         if (doCoreFxTesting) {
2221                             if (scenario == 'corefx_innerloop') {
2222                                 // Create CORE_ROOT and testhost
2223                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2224                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2225
2226                                 // Archive and process (only) the test results
2227                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2228                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2229                             }
2230                             else {
2231                               def workspaceRelativeFxRoot = "_/fx"
2232                               def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2233                               def fxBranch = getFxBranch(branch)
2234
2235                               buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2236
2237                               // Archive and process (only) the test results
2238                               Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2239                               Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2240
2241                               //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2242                               Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2243                               Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2244                             }
2245                         }
2246                         else if (isGcReliabilityFramework(scenario)) {
2247                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2248                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2249                         }
2250                         else {
2251                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2252                         }
2253                     } // end if (!isBuildOnly)
2254
2255                     if (!doCoreFxTesting) {
2256                         // Run the rest of the build
2257                         // Build the mscorlib for the other OS's
2258                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2259                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2260
2261                         if (arch == 'x64') {
2262                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2263                         }
2264
2265                         // Zip up the tests directory so that we don't use so much space/time copying
2266                         // 10s of thousands of files around.
2267                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2268
2269                         if (!isJitStressScenario(scenario)) {
2270                             // For Windows, pull full test results and test drops for x86/x64.
2271                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2272                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2273                         }
2274
2275                         if (scenario == 'jitdiff') {
2276                             // retrieve jit-dasm output for base commit, and run jit-diff
2277                             if (!isBuildOnly) {
2278                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2279                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2280                             }
2281                         }
2282
2283                         if (!isBuildOnly) {
2284                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2285                         }
2286                     }
2287                     break
2288                 case 'arm':
2289                     assert isArmWindowsScenario(scenario)
2290
2291                     def buildArchitecture = 'arm'
2292                     def buildOpts = ''
2293
2294                     if (doCoreFxTesting) {
2295                         // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx
2296                         // package available, to determine the correct corefx version git commit hash, and we need to build the tests before
2297                         // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen.
2298                         //
2299                         // buildOpts += ' skiptests';
2300                         buildOpts += " -priority=0"
2301                     } else {
2302                         buildOpts += " -priority=${priority}"
2303                     }
2304
2305                     // This is now a build only job. Do not run tests. Use the flow job.
2306                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}"
2307
2308                     if (doCoreFxTesting) {
2309                         assert isBuildOnly
2310                         assert architecture == 'arm'
2311
2312                         // Generate the test layout because it restores the corefx package which allows run-corefx-tests.py
2313                         // to determine the correct matching corefx version git commit hash.
2314                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} GenerateLayoutOnly"
2315
2316                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2317                         def envScriptPath = ''
2318                         def buildCommandsStr = ''
2319                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2320                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2321                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2322                         envScriptFinalize(os, envScriptPath)
2323                         buildCommands += buildCommandsStr
2324
2325                         def workspaceRelativeFxRootLinux = "_/fx"
2326                         def workspaceRelativeFxRootWin = "_\\fx"
2327                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2328                         def fxBranch = getFxBranch(branch)
2329
2330                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2331
2332                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2333                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2334                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2335
2336                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2337                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2338                     } else {
2339                         // Zip up the tests directory so that we don't use so much space/time copying
2340                         // 10s of thousands of files around.
2341                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2342
2343                         // Add archival.
2344                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2345                     }
2346                     break
2347                 case 'arm64':
2348                     assert isArmWindowsScenario(scenario)
2349
2350                     // This is now a build only job. Do not run tests. Use the flow job.
2351                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
2352
2353                     // Zip up the tests directory so that we don't use so much space/time copying
2354                     // 10s of thousands of files around.
2355                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2356
2357                     // Add archival.
2358                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2359                     break
2360                 default:
2361                     println("Unknown architecture: ${architecture}");
2362                     assert false
2363                     break
2364             }
2365             break
2366         // end case 'Windows_NT'; editor brace matching: }
2367         case 'Ubuntu':
2368         case 'Ubuntu16.04':
2369         case 'Ubuntu16.10':
2370         case 'Debian8.4':
2371         case 'OSX10.12':
2372         case 'CentOS7.1':
2373         case 'RHEL7.2':
2374         case 'Tizen':
2375         case 'Fedora24': // editor brace matching: {
2376             switch (architecture) {
2377                 case 'x64':
2378                 case 'x86':
2379                     if (architecture == 'x86' && os == 'Ubuntu') {
2380                         // build and PAL test
2381                         def dockerImage = getDockerImageName(architecture, os, true)
2382                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2383                         dockerImage = getDockerImageName(architecture, os, false)
2384                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2385                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2386                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2387                         break
2388                     }
2389
2390                     if (scenario == 'formatting') {
2391                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2392                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2393                         break
2394                     }
2395
2396                     if (scenario == 'illink') {
2397                         assert(os == 'Ubuntu')
2398                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2399                     }
2400
2401                     if (!doCoreFxTesting) {
2402                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2403                         // only on supported OS platforms.
2404                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2405                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2406
2407                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2408                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2409
2410                         // Basic archiving of the build
2411                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2412                         // And pal tests
2413                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2414                     }
2415                     else {
2416                         // Corefx stress testing
2417                         assert os == 'Ubuntu'
2418                         assert architecture == 'x64'
2419                         assert lowerConfiguration == 'checked'
2420                         assert isJitStressScenario(scenario)
2421
2422                         // Build coreclr
2423                         buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2424
2425                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2426
2427                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2428                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2429                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2430                         buildCommands += envScriptCmds
2431
2432                         // Build and text corefx
2433                         def workspaceRelativeFxRoot = "_/fx"
2434                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2435                         def fxBranch = getFxBranch(branch)
2436
2437                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2438
2439                         // Archive and process (only) the test results
2440                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2441                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2442                     }
2443                     break
2444                 case 'armem':
2445                     // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
2446                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
2447
2448                     // default values for Ubuntu
2449                     def arm_abi = "arm"
2450                     def linuxCodeName = "trusty"
2451                     if (os == 'Ubuntu16.04') {
2452                         linuxCodeName = "xenial"
2453                     }
2454                     else if (os == 'Tizen') {
2455                         arm_abi = "armel"
2456                         linuxCodeName = "tizen"
2457                     }
2458
2459                     // Unzip the Windows test binaries first. Exit with 0
2460                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2461
2462                     // Unpack the corefx binaries
2463                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2464                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2465                     if (os != 'Tizen') {
2466                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2467                     }
2468                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2469                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2470                     if (os == 'Ubuntu') {
2471                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2472                     }
2473
2474                     // Call the ARM CI script to cross build and test using docker
2475                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2476                     --mode=docker \\
2477                     --${arm_abi} \\
2478                     --linuxCodeName=${linuxCodeName} \\
2479                     --buildConfig=${lowerConfiguration} \\
2480                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2481                     --coreFxBinDir=./bin/CoreFxBinDir \\
2482                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2483
2484                     // Basic archiving of the build, no pal tests
2485                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2486                     break
2487                 case 'arm64':
2488                 case 'arm':
2489                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2490                     assert (os == 'Ubuntu')
2491
2492                     // Add some useful information to the log file. Ignore return codes.
2493                     buildCommands += "uname -a || true"
2494
2495                     def additionalOpts = ""
2496                     if (architecture == 'arm') {
2497                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2498                     }
2499
2500                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2501                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2502                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2503                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2504                     // ZIP up the generated CoreFX runtime and tests.
2505
2506                     def dockerImage = getDockerImageName(architecture, os, true)
2507                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2508
2509                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2510
2511                     if (doCoreFxTesting) {
2512                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2513
2514                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2515                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2516                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2517                         buildCommands += envScriptCmds
2518
2519                         // Build and text corefx
2520                         def workspaceRelativeFxRootLinux = "_/fx"
2521                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2522                         def fxBranch = getFxBranch(branch)
2523
2524                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2525
2526                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2527                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2528                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2529
2530                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2531                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2532                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2533                     }
2534                     else {
2535                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2536                         // download the appropriate CoreFX packages.
2537                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2538                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2539
2540                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2541
2542                         // ZIP up for the test job (created in the flow job code):
2543                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2544                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2545                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2546                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2547
2548                         // These commands are assumed to be run from the root of the workspace.
2549                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2550                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2551
2552                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2553                     }
2554
2555                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2556                     // the cleanup task in Jenkins can't remove.
2557                     newJob.with {
2558                         publishers {
2559                             azureVMAgentPostBuildAction {
2560                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2561                             }
2562                         }
2563                     }
2564                     break
2565                 default:
2566                     println("Unknown architecture: ${architecture}");
2567                     assert false
2568                     break
2569             }
2570             break
2571         // editor brace matching: }
2572         default:
2573             println("Unknown os: ${os}");
2574             assert false
2575             break
2576     } // os
2577
2578     return buildCommands
2579 }
2580
2581 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2582 // Returns true if the job should be generated.
2583 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2584 {
2585     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2586     // All other scenarios do Pri-1 testing.
2587     if (scenario == 'innerloop' && !isPR) {
2588         return false
2589     }
2590
2591     // Run basic corefx tests only on PR-triggered jobs
2592     // Runs under Release and Checked
2593     if (scenario == 'corefx_innerloop' && !isPR) {
2594         return false
2595     }
2596
2597     // Tizen is only supported for armem architecture
2598     if (os == 'Tizen' && architecture != 'armem') {
2599         return false
2600     }
2601
2602     // Filter based on architecture.
2603
2604     switch (architecture) {
2605         case 'arm64':
2606         case 'arm':
2607             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2608                 return false
2609             }
2610             break
2611         case 'armem':
2612             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) {
2613                 return false
2614             }
2615             break
2616         case 'x86_arm_altjit':
2617         case 'x64_arm64_altjit':
2618             if (os != 'Windows_NT') {
2619                 return false
2620             }
2621             break
2622         case 'x86':
2623             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2624                 return false
2625             }
2626             break
2627         case 'x64':
2628             // Everything implemented
2629             break
2630         default:
2631             println("Unknown architecture: ${architecture}")
2632             assert false
2633             break
2634     }
2635
2636     // Which (Windows) build only jobs are required?
2637
2638     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2639
2640     if (isBuildOnly) {
2641         switch (architecture) {
2642             case 'arm':
2643                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2644                 if (!isCoreFxScenario(scenario)) {
2645                     return false
2646                 }
2647                 break
2648             case 'x64':
2649             case 'x86':
2650                 if (!isNormalOrInnerloop) {
2651                     return false
2652                 }
2653                 break
2654             default:
2655                 return false
2656         }
2657     }
2658
2659     // Filter based on scenario.
2660
2661     if (isJitStressScenario(scenario)) {
2662         if (configuration != 'Checked') {
2663             return false
2664         }
2665
2666         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
2667         if (!isEnabledOS) {
2668             return false
2669         }
2670
2671         switch (architecture) {
2672             case 'x64':
2673             case 'x86_arm_altjit':
2674             case 'x64_arm64_altjit':
2675                 break
2676
2677             case 'x86':
2678                 // x86 ubuntu: no stress modes
2679                 if (os == 'Ubuntu') {
2680                     return false
2681                 }
2682                 break
2683
2684             case 'arm':
2685                 // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that.
2686                 // No "regular" Windows arm corefx jobs, e.g.
2687                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2688                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2689                 if (os == 'Windows_NT') {
2690                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2691                         return false
2692                     }
2693                 }
2694                 else {
2695                     if (!isCoreFxScenario(scenario)) {
2696                         return false
2697                     }
2698                 }
2699                 break
2700
2701             case 'arm64':
2702                 if (os == 'Windows_NT') {
2703                     return false
2704                 }
2705                 else {
2706                     if (!isCoreFxScenario(scenario)) {
2707                         return false
2708                     }
2709                 }
2710                 break
2711
2712             default:
2713                 // arm64: stress is handled through flow jobs.
2714                 // armem: no stress jobs for ARM emulator.
2715                 return false
2716         }
2717     }
2718     else if (isR2RScenario(scenario)) {
2719         if (os != 'Windows_NT') {
2720             return false
2721         }
2722         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2723         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2724             return false
2725         }
2726     }
2727     else {
2728         // Skip scenarios
2729         switch (scenario) {
2730             case 'ilrt':
2731                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2732                 // the default scenario build
2733                 if (os != 'Windows_NT') {
2734                     return false
2735                 }
2736                 // Only x64 for now
2737                 if (architecture != 'x64') {
2738                     return false
2739                 }
2740                 // Release only
2741                 if (configuration != 'Release') {
2742                     return false
2743                 }
2744                 break
2745             case 'jitdiff':
2746                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2747                     return false
2748                 }
2749                 if (architecture != 'x64') {
2750                     return false
2751                 }
2752                 if (configuration != 'Checked') {
2753                     return false
2754                 }
2755                 break
2756             case 'longgc':
2757             case 'gcsimulator':
2758                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2759                     return false
2760                 }
2761                 if (architecture != 'x64') {
2762                     return false
2763                 }
2764                 if (configuration != 'Release') {
2765                     return false
2766                 }
2767                 break
2768             case 'gc_reliability_framework':
2769             case 'standalone_gc':
2770                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2771                     return false
2772                 }
2773
2774                 if (architecture != 'x64') {
2775                     return false
2776                 }
2777
2778                 if (configuration != 'Release' && configuration != 'Checked') {
2779                     return false
2780                 }
2781                 break
2782             // We only run Windows and Ubuntu x64 Checked for formatting right now
2783             case 'formatting':
2784                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2785                     return false
2786                 }
2787                 if (architecture != 'x64') {
2788                     return false
2789                 }
2790                 if (configuration != 'Checked') {
2791                     return false
2792                 }
2793                 break
2794             case 'illink':
2795                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2796                     return false
2797                 }
2798                 if (architecture != 'x64' && architecture != 'x86') {
2799                     return false
2800                 }
2801                 break
2802             case 'normal':
2803                 // Nothing skipped
2804                 break
2805             case 'innerloop':
2806                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2807                     return false
2808                 }
2809                 break
2810             case 'corefx_innerloop':
2811                 if (os != 'Windows_NT'|| architecture != 'x64') {
2812                     return false
2813                 }
2814                 if(configuration != 'Release' && configuration != 'Checked') {
2815                     return false
2816                 }
2817                 break
2818             default:
2819                 println("Unknown scenario: ${scenario}")
2820                 assert false
2821                 break
2822         }
2823     }
2824
2825     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2826     // runtime behavior, not compile-time behavior, are not interesting.
2827     switch (architecture) {
2828         case 'x86_arm_altjit':
2829         case 'x64_arm64_altjit':
2830             if (isGCStressRelatedTesting(scenario)) {
2831                 return false
2832             }
2833             break
2834         default:
2835             break
2836     }
2837
2838     // The job was not filtered out, so we should generate it!
2839     return true
2840 }
2841
2842 Constants.allScenarios.each { scenario ->
2843     [true, false].each { isPR ->
2844         Constants.architectureList.each { architecture ->
2845             Constants.configurationList.each { configuration ->
2846                 Constants.osList.each { os ->
2847                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2848                     // and reset the os to Windows_NT
2849                     def isBuildOnly = false
2850                     if (os == 'Windows_NT_BuildOnly') {
2851                         isBuildOnly = true
2852                         os = 'Windows_NT'
2853                     }
2854
2855                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2856                         return
2857                     }
2858
2859                     // Calculate names
2860                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2861                     def folderName = getJobFolder(scenario)
2862
2863                     // Create the new job
2864                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2865
2866                     // Should we add corefx_innerloop to views?
2867                     addToViews(newJob, isPR, architecture, os)
2868
2869                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2870
2871                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2872                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2873                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2874
2875                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2876                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2877                     // artifacts produced.
2878                     if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2879                         // Define the Windows Tests and Corefx build job names
2880                         def lowerConfiguration = configuration.toLowerCase()
2881                         def WindowsTestsName = projectFolder + '/' +
2882                                                Utilities.getFullJobName(project,
2883                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2884                                                                         false)
2885                         def fxBranch = getFxBranch(branch)
2886                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2887                                            Utilities.getFolderName(fxBranch)
2888
2889                         def arm_abi = 'arm'
2890                         def corefx_os = 'linux'
2891                         if (os == 'Tizen') {
2892                             arm_abi = 'armel'
2893                             corefx_os = 'tizen'
2894                         }
2895
2896                         // Let's use release CoreFX to test checked CoreCLR,
2897                         // because we do not generate checked CoreFX in CoreFX CI yet.
2898                         def corefx_lowerConfiguration = lowerConfiguration
2899                         if (lowerConfiguration == 'checked') {
2900                             corefx_lowerConfiguration = 'release'
2901                         }
2902
2903                         // Copy the Windows test binaries and the Corefx build binaries
2904                         newJob.with {
2905                             steps {
2906                                 copyArtifacts(WindowsTestsName) {
2907                                     includePatterns('bin/tests/tests.zip')
2908                                     buildSelector {
2909                                         latestSuccessful(true)
2910                                     }
2911                                 }
2912                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2913                                     includePatterns('bin/build.tar.gz')
2914                                     buildSelector {
2915                                         latestSuccessful(true)
2916                                     }
2917                                 }
2918                             } // steps
2919                         } // newJob.with
2920                     }
2921
2922                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2923
2924                     newJob.with {
2925                         steps {
2926                             if (os == 'Windows_NT') {
2927                                 buildCommands.each { buildCommand ->
2928                                     batchFile(buildCommand)
2929                                 }
2930                             }
2931                             else {
2932                                 buildCommands.each { buildCommand ->
2933                                     shell(buildCommand)
2934                                 }
2935                             }
2936                         } // steps
2937                     } // newJob.with
2938
2939                 } // os
2940             } // configuration
2941         } // architecture
2942     } // isPR
2943 } // scenario
2944
2945 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2946 // Returns the newly created job.
2947 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2948 {
2949     def osGroup = getOSGroup(os)
2950     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2951
2952     def jobFolder = getJobFolder(scenario)
2953     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2954         parameters {
2955             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2956         }
2957
2958         steps {
2959             // Set up the copies
2960
2961             // Coreclr build we are trying to test
2962             //
2963             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2964
2965             copyArtifacts(inputCoreCLRBuildName) {
2966                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2967                 buildSelector {
2968                     buildNumber('${CORECLR_BUILD}')
2969                 }
2970             }
2971
2972             if (isCoreFxScenario(scenario)) {
2973
2974                 // Only arm supported for corefx testing now.
2975                 assert architecture == 'arm'
2976
2977                 // Unzip CoreFx runtime
2978                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')\"")
2979
2980                 // Unzip CoreFx tests.
2981                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
2982
2983                 // Add the script to run the corefx tests
2984                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm"
2985                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
2986                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt"
2987                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}")
2988
2989             } else { // !isCoreFxScenario(scenario)
2990
2991                 // Unzip tests.
2992                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
2993
2994                 def buildCommands = ""
2995
2996                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2997                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2998                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2999
3000                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3001                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3002
3003                 // For all jobs
3004                 addEnvVariable("CORE_ROOT", coreRootLocation)
3005                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3006                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3007
3008                 // If we are running a stress mode, we'll set those variables as well
3009                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3010                     def stressValues = null
3011                     if (isJitStressScenario(scenario)) {
3012                         stressValues = Constants.jitStressModeScenarios[scenario]
3013                     }
3014                     else {
3015                         stressValues = Constants.r2rStressScenarios[scenario]
3016                     }
3017
3018                     stressValues.each { key, value ->
3019                         addEnvVariable(key, value)
3020                     }
3021                 }
3022
3023                 if (isR2RScenario(scenario)) {
3024                     // Crossgen the framework assemblies.
3025                     buildCommands += """
3026 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3027 @goto skip_PrecompileAssembly
3028
3029 :PrecompileAssembly
3030 @REM Skip mscorlib since it is already precompiled.
3031 @if /I "%3" == "mscorlib.dll" exit /b 0
3032 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3033
3034 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3035 @if "%errorlevel%" == "-2146230517" (
3036     echo %2 is not a managed assembly.
3037 ) else if "%errorlevel%" == "-2146234344" (
3038     echo %2 is not a managed assembly.
3039 ) else if %errorlevel% neq 0 (
3040     echo Unable to precompile %2
3041 ) else (
3042     echo Precompiled %2
3043 )
3044 @exit /b 0
3045
3046 :skip_PrecompileAssembly
3047 """
3048
3049                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3050                     // crossgen on tests before running them.
3051                     addEnvVariable("RunCrossGen", "true")
3052                 } // isR2RScenario(scenario)
3053
3054                 // Create the smarty command
3055                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3056                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3057                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3058                 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3059
3060                 // Exclude tests based on scenario.
3061                 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3062                     addArchSpecificExclude(architecture, excludeTag)
3063                 }
3064
3065                 // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1.
3066                 if (scenario == 'innerloop') {
3067                     addExclude("pri1")
3068                 }
3069
3070                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3071                 // TODO: We should create a "long running" job that runs these with a longer timeout.
3072                 addExclude("LONG_RUNNING")
3073
3074                 smartyCommand += "/lstFile Tests.lst"
3075
3076                 def testListArch = [
3077                     'arm64': 'arm64',
3078                     'arm': 'arm'
3079                 ]
3080
3081                 def archLocation = testListArch[architecture]
3082
3083                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3084
3085                 if (architecture == "arm64") {
3086                     addCommand("copy C:\\Jenkins\\vcruntime140.dll bin\\tests\\${osGroup}.${architecture}.${configuration}\\Tests\\Core_Root")
3087                     addCommand("copy C:\\Jenkins\\vcruntime140d.dll bin\\tests\\${osGroup}.${architecture}.${configuration}\\Tests\\Core_Root")
3088                 }
3089
3090                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3091                 addCommand("${smartyCommand}")
3092
3093                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3094                 // However, we also need to remove all the variables that were set during this batch file, so we
3095                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3096                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3097                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3098                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3099                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3100                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3101
3102                 // ZIP up the smarty output, no matter what the smarty result.
3103                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3104
3105                 addCommand("echo %errorlevel%")
3106                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3107
3108                 // Use the smarty errorlevel as the script errorlevel.
3109                 addCommand("exit /b %__save_smarty_errorlevel%")
3110
3111                 batchFile(buildCommands)
3112             } // non-corefx testing
3113         } // steps
3114     } // job
3115
3116     if (!isCoreFxScenario(scenario)) {
3117         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3118
3119         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3120         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3121         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3122         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3123     }
3124
3125     return newJob
3126 }
3127
3128 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3129 // E.g., non-Windows tests.
3130 // Returns the newly created job.
3131 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3132 {
3133     def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
3134     def doCoreFxTesting = isCoreFxScenario(scenario)
3135
3136     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3137
3138     def osGroup = getOSGroup(os)
3139     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3140
3141     def testOpts = ''
3142     def useServerGC = false
3143
3144     // Enable Server GC for Ubuntu PR builds
3145     // REVIEW: why? Does this apply to all architectures? Why only PR?
3146     if (os == 'Ubuntu' && isPR) {
3147         testOpts += ' --useServerGC'
3148         useServerGC = true
3149     }
3150
3151     if (isR2RScenario(scenario)) {
3152
3153         testOpts += ' --crossgen --runcrossgentests'
3154
3155         if (scenario == 'r2r_jitstress1') {
3156             testOpts += ' --jitstress=1'
3157         }
3158         else if (scenario == 'r2r_jitstress2') {
3159             testOpts += ' --jitstress=2'
3160         }
3161         else if (scenario == 'r2r_jitstressregs1') {
3162             testOpts += ' --jitstressregs=1'
3163         }
3164         else if (scenario == 'r2r_jitstressregs2') {
3165             testOpts += ' --jitstressregs=2'
3166         }
3167         else if (scenario == 'r2r_jitstressregs3') {
3168             testOpts += ' --jitstressregs=3'
3169         }
3170         else if (scenario == 'r2r_jitstressregs4') {
3171             testOpts += ' --jitstressregs=4'
3172         }
3173         else if (scenario == 'r2r_jitstressregs8') {
3174             testOpts += ' --jitstressregs=8'
3175         }
3176         else if (scenario == 'r2r_jitstressregs0x10') {
3177             testOpts += ' --jitstressregs=0x10'
3178         }
3179         else if (scenario == 'r2r_jitstressregs0x80') {
3180             testOpts += ' --jitstressregs=0x80'
3181         }
3182         else if (scenario == 'r2r_jitstressregs0x1000') {
3183             testOpts += ' --jitstressregs=0x1000'
3184         }
3185         else if (scenario == 'r2r_jitminopts') {
3186             testOpts += ' --jitminopts'
3187         }
3188         else if (scenario == 'r2r_jitforcerelocs') {
3189             testOpts += ' --jitforcerelocs'
3190         }
3191         else if (scenario == 'r2r_gcstress15') {
3192             testOpts += ' --gcstresslevel=0xF'
3193         }
3194     }
3195     else if (scenario == 'jitdiff') {
3196         testOpts += ' --jitdisasm --crossgen'
3197     }
3198     else if (scenario == 'illink') {
3199         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3200     }
3201     else if (isLongGc(scenario)) {
3202         // Long GC tests behave very poorly when they are not
3203         // the only test running (many of them allocate until OOM).
3204         testOpts += ' --sequential'
3205
3206         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3207         // for running long GC and GCSimulator tests, respectively. We don't use them
3208         // here because using a playlist file produces much more readable output on the CI machines
3209         // and reduces running time.
3210         //
3211         // The Long GC playlist contains all of the tests that are
3212         // going to be run. The GCSimulator playlist contains all of
3213         // the GC simulator tests.
3214         if (scenario == 'longgc') {
3215             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3216         }
3217         else if (scenario == 'gcsimulator') {
3218             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3219         }
3220     }
3221     else if (isGcReliabilityFramework(scenario)) {
3222         testOpts += ' --build-overlay-only'
3223     }
3224     else if (scenario == 'standalone_gc') {
3225         if (osGroup == 'OSX') {
3226             testOpts += ' --gcname=libclrgc.dylib'
3227         }
3228         else if (osGroup == 'Linux') {
3229             testOpts += ' --gcname=libclrgc.so'
3230         }
3231         else {
3232             println("Unexpected OS group: ${osGroup} for os ${os}")
3233             assert false
3234         }
3235     }
3236
3237     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3238     // will be null in this case.
3239
3240     def jobFolder = getJobFolder(scenario)
3241     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3242         parameters {
3243             if (inputTestsBuildName != null) {
3244                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3245             }
3246             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3247         }
3248
3249         steps {
3250             // Set up the copies
3251
3252             // Coreclr build containing the tests and mscorlib
3253             // pri1 jobs still need to copy windows_nt built tests
3254             if (inputTestsBuildName != null) {
3255                 copyArtifacts(inputTestsBuildName) {
3256                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3257                     buildSelector {
3258                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3259                     }
3260                 }
3261             }
3262
3263             // Coreclr build we are trying to test
3264             //
3265             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3266
3267             copyArtifacts(inputCoreCLRBuildName) {
3268                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3269                 buildSelector {
3270                     buildNumber('${CORECLR_BUILD}')
3271                 }
3272             }
3273
3274             if (isUbuntuArmJob) {
3275                 // Add some useful information to the log file. Ignore return codes.
3276                 shell("uname -a || true")
3277             }
3278
3279             if (architecture == 'x86') {
3280                 shell("mkdir ./bin/CoreFxNative")
3281
3282                 def fxBranch = getFxBranch(branch)
3283                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3284
3285                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3286                     includePatterns('bin/build.tar.gz')
3287                     targetDirectory('bin/CoreFxNative')
3288                     buildSelector {
3289                         latestSuccessful(true)
3290                     }
3291                 }
3292
3293                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3294             }
3295
3296             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3297             if (doCoreFxTesting) {
3298                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3299                 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3300             }
3301             else {
3302                 // Unzip the tests first.  Exit with 0
3303                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3304                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3305             }
3306
3307             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3308             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3309             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3310             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3311             // copied correctly.
3312             if (!doCoreFxTesting) {
3313                 if (isUbuntuArmJob) {
3314                     def lowerConfiguration = configuration.toLowerCase()
3315                     shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3316                     shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3317                 }
3318                 else {
3319                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3320                 }
3321             }
3322
3323             // Execute the tests
3324             def runDocker = isNeedDocker(architecture, os, false)
3325             def dockerPrefix = ""
3326             def dockerCmd = ""
3327             if (runDocker) {
3328                 def dockerImage = getDockerImageName(architecture, os, false)
3329                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3330                 dockerCmd = dockerPrefix + "${dockerImage} "
3331             }
3332
3333             // If we are running a stress mode, we'll set those variables first.
3334             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3335             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3336                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3337                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3338                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3339                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3340                 shell("${envScriptCmds}")
3341                 testOpts += " --test-env=${scriptFileName}"
3342             }
3343
3344             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3345             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3346             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3347             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3348             if (isGCStressRelatedTesting(scenario)) {
3349                 if (architecture == 'x64') {
3350                     shell('./init-tools.sh')
3351                 }
3352             }
3353
3354             if (doCoreFxTesting) {
3355                 shell("""\
3356 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3357             }
3358             else {
3359                 def runScript = "${dockerCmd}./tests/runtest.sh"
3360
3361                 shell("""\
3362 ${runScript} \\
3363     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3364     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3365     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3366     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3367             }
3368
3369             if (isGcReliabilityFramework(scenario)) {
3370                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3371                 if (useServerGC) {
3372                     if (runDocker) {
3373                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3374                     }
3375                     else {
3376                         shell("export COMPlus_gcServer=1")
3377                     }
3378                 }
3379
3380                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3381             }
3382         } // steps
3383     } // job
3384
3385     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3386     if (os in ['Ubuntu']) {
3387         SummaryBuilder summaries = new SummaryBuilder()
3388         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3389         summaries.emit(newJob)
3390     }
3391
3392     if (doCoreFxTesting) {
3393         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3394         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3395     }
3396     else {
3397         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt")
3398         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3399     }
3400
3401     return newJob
3402 }
3403
3404 // Create a test job that will be used by a flow job.
3405 // Returns the newly created job.
3406 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3407 {
3408     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3409
3410     def newJob = null
3411     if (windowsArmJob) {
3412         assert inputTestsBuildName == null
3413         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3414     } else {
3415         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3416     }
3417
3418     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3419
3420     addToViews(newJob, isPR, architecture, os)
3421
3422     if (scenario == 'jitdiff') {
3423         def osGroup = getOSGroup(os)
3424         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3425     }
3426
3427     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3428     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3429
3430     return newJob
3431 }
3432
3433 // Create a flow job to tie together a build job with the given test job.
3434 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3435 // Returns the new flow job.
3436 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3437 {
3438     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3439     // Linux CoreCLR test
3440     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3441     def jobFolder = getJobFolder(scenario)
3442
3443     def newFlowJob = null
3444
3445     if (inputTestsBuildName == null) {
3446         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3447                         buildFlow("""\
3448 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3449
3450 // And then build the test build
3451 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3452 """)
3453         }
3454         JobReport.Report.addReference(inputCoreCLRBuildName)
3455         JobReport.Report.addReference(fullTestJobName)
3456     }
3457     else {
3458         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3459                         buildFlow("""\
3460 // Build the input jobs in parallel
3461 parallel (
3462 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3463 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3464 )
3465
3466 // And then build the test build
3467 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3468                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3469 """)
3470         }
3471         JobReport.Report.addReference(inputCoreCLRBuildName)
3472         JobReport.Report.addReference(inputTestsBuildName)
3473         JobReport.Report.addReference(fullTestJobName)
3474     }
3475
3476     addToViews(newFlowJob, isPR, architecture, os)
3477
3478     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3479
3480     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3481     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3482
3483     return newFlowJob
3484 }
3485
3486 // Determine if we should generate a flow job for the given parameters.
3487 // Returns true if the job should be generated.
3488 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3489 {
3490     // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3491     // All other scenarios do Pri-1 testing.
3492     if (scenario == 'innerloop' && !isPR) {
3493         return false
3494     }
3495
3496     if (scenario == 'corefx_innerloop') {
3497         return false
3498     }
3499
3500     // Filter based on OS and architecture.
3501
3502     switch (architecture) {
3503         case 'arm64':
3504             if (os != "Ubuntu" && os != "Windows_NT") {
3505                 return false
3506             }
3507             break
3508         case 'arm':
3509             if (os != "Ubuntu" && os != "Windows_NT") {
3510                 return false
3511             }
3512             break
3513         case 'x86':
3514             if (os != "Ubuntu") {
3515                 return false
3516             }
3517             break
3518         case 'x64':
3519             if (!(os in Constants.crossList)) {
3520                 return false
3521             }
3522             if (os == "Windows_NT") {
3523                 return false
3524             }
3525             break
3526         case 'armem':
3527         case 'x86_arm_altjit':
3528         case 'x64_arm64_altjit':
3529             // No flow jobs
3530             return false
3531         default:
3532             println("Unknown architecture: ${architecture}")
3533             assert false
3534             break
3535     }
3536
3537     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3538
3539     // Filter based on scenario in OS.
3540
3541     if (os == 'Windows_NT') {
3542         if (!isArmWindowsScenario(scenario)) {
3543             return false
3544         }
3545     }
3546     else {
3547         // Non-Windows
3548         if (architecture == 'arm64') {
3549             if (!(scenario in Constants.validLinuxArmScenarios)) {
3550                 return false
3551             }
3552         }
3553         else if (architecture == 'arm') {
3554             if (!(scenario in Constants.validLinuxArmScenarios)) {
3555                 return false
3556             }
3557         }
3558         else if (architecture == 'x86') {
3559             // Linux/x86 only want innerloop and default test
3560             if (!isNormalOrInnerloop) {
3561                 return false
3562             }
3563         }
3564         else if (architecture == 'x64') {
3565             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3566             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3567             // build from a Windows machine.
3568             if (isCoreFxScenario(scenario)) {
3569                 return false
3570             }
3571         }
3572     }
3573
3574     // For CentOS, we only want Checked/Release builds.
3575     if (os == 'CentOS7.1') {
3576         if (configuration != 'Checked' && configuration != 'Release') {
3577             return false
3578         }
3579         if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
3580             return false
3581         }
3582     }
3583
3584     // For RedHat and Debian, we only do Release builds.
3585     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3586         if (configuration != 'Release') {
3587             return false
3588         }
3589         if (!isNormalOrInnerloop) {
3590             return false
3591         }
3592     }
3593
3594     // Next, filter based on scenario.
3595
3596     if (isJitStressScenario(scenario)) {
3597         if (configuration != 'Checked') {
3598             return false
3599         }
3600
3601         // On Windows, CoreFx tests currently not implemented for ARM64.
3602         if (isCoreFxScenario(scenario) && (os == 'Windows_NT') && (architecture == 'arm64')) {
3603             return false
3604         }
3605     }
3606     else if (isR2RBaselineScenario(scenario)) {
3607         if (configuration != 'Checked' && configuration != 'Release') {
3608             return false
3609         }
3610     }
3611     else if (isR2RStressScenario(scenario)) {
3612         if (configuration != 'Checked') {
3613             return false
3614         }
3615     }
3616     else {
3617         // Skip scenarios
3618         switch (scenario) {
3619             case 'ilrt':
3620             case 'longgc':
3621             case 'gcsimulator':
3622                 // Long GC tests take a long time on non-Release builds
3623                 // ilrt is also Release only
3624                 if (configuration != 'Release') {
3625                     return false
3626                 }
3627                 break
3628
3629             case 'jitdiff':
3630                 if (configuration != 'Checked') {
3631                     return false
3632                 }
3633                 break
3634
3635             case 'gc_reliability_framework':
3636             case 'standalone_gc':
3637                 if (configuration != 'Release' && configuration != 'Checked') {
3638                     return false
3639                 }
3640                 break
3641
3642             case 'formatting':
3643                 return false
3644             case 'illink':
3645                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3646                     return false
3647                 }
3648                 break
3649
3650             case 'normal':
3651                 // Nothing skipped
3652                 break
3653
3654             case 'innerloop':
3655                 // Nothing skipped
3656                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3657                     return false
3658                 }
3659                 break
3660
3661             default:
3662                 println("Unknown scenario: ${scenario}")
3663                 assert false
3664                 break
3665         }
3666     }
3667
3668     // The job was not filtered out, so we should generate it!
3669     return true
3670 }
3671
3672 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3673 Constants.allScenarios.each { scenario ->
3674     [true, false].each { isPR ->
3675         Constants.architectureList.each { architecture ->
3676             Constants.configurationList.each { configuration ->
3677                 Constants.osList.each { os ->
3678
3679                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3680                         return
3681                     }
3682
3683                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3684                     def doCoreFxTesting = isCoreFxScenario(scenario)
3685
3686                     // Figure out the job name of the CoreCLR build the test will depend on.
3687
3688                     def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3689                     def inputCoreCLRBuildIsBuildOnly = false
3690                     if (doCoreFxTesting) {
3691                         // Every CoreFx test depends on its own unique build.
3692                         inputCoreCLRBuildScenario = scenario
3693                         if (windowsArmJob) {
3694                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3695                             inputCoreCLRBuildIsBuildOnly = true
3696                         }
3697                     }
3698                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3699                     def inputCoreCLRBuildName = projectFolder + '/' +
3700                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3701
3702                     // Figure out the name of the build job that the test job will depend on.
3703                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3704                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3705                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3706                     // CoreFX build creates the tests.
3707
3708                     def inputTestsBuildName = null
3709
3710                     if (!windowsArmJob && !doCoreFxTesting) {
3711                         def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal'
3712
3713                         def inputTestsBuildArch = architecture
3714                         if (architecture == "arm64") {
3715                             // Use the x64 test build for arm64 unix
3716                             inputTestsBuildArch = "x64"
3717                         }
3718                         else if (architecture == "arm") {
3719                             // Use the x86 test build for arm unix
3720                             inputTestsBuildArch = "x86"
3721                         }
3722
3723                         def inputTestsBuildIsBuildOnly = true
3724
3725                         inputTestsBuildName = projectFolder + '/' +
3726                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3727                     }
3728
3729                     // =============================================================================================
3730                     // Create the test job
3731                     // =============================================================================================
3732
3733                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3734
3735                     // =============================================================================================
3736                     // Create a build flow to join together the build and tests required to run this test.
3737                     // =============================================================================================
3738
3739                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3740                         // Do not create the flow job for RHEL jobs.
3741                         return
3742                     }
3743
3744                     def fullTestJobName = projectFolder + '/' + testJob.name
3745                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3746
3747                 } // os
3748             } // configuration
3749         } // architecture
3750     } // isPR
3751 } // scenario
3752
3753 JobReport.Report.generateJobReport(out)
3754
3755 // Make the call to generate the help job
3756 Utilities.createHelperJob(this, project, branch,
3757     "Welcome to the ${project} Repository",  // This is prepended to the help message
3758     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
3759
3760 Utilities.addCROSSCheck(this, project, branch)