netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu'      : 'Linux',
  21                       'Ubuntu16.04' : 'Linux',
  22                       'Ubuntu16.10' : 'Linux',
  23                       'RHEL7.2'     : 'Linux',
  24                       'Debian8.4'   : 'Linux',
  25                       'Fedora24'    : 'Linux',
  26                       'CentOS7.1'   : 'Linux',
  27                       'Tizen'       : 'Linux',
  28                       'OSX10.12'    : 'OSX',
  29                       'Windows_NT'  : 'Windows_NT']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 def static getCrossArchitectures(def os, def architecture, def scenario) {
  36     switch (architecture) {
  37         case 'arm':
  38             return ['x86','x64']
  39
  40         case 'arm64':
  41             return ['x64']
  42     }
  43
  44     assert false
  45 }
  46
  47 // We use this class (vs variables) so that the static functions can access data here.
  48 class Constants {
  49
  50     // We have very limited Windows ARM64 hardware (used for ARM/ARM64 testing) and Linux/arm32 and Linux/arm64 hardware.
  51     // So only allow certain branches to use it.
  52     def static LimitedHardwareBranches = [
  53                'master']
  54
  55     // Innerloop build OS's
  56     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  57     // test execution in the build flow runs.  It generates the exact same build
  58     // as Windows_NT but without running the tests.
  59     def static osList = [
  60                'Ubuntu',
  61                'Debian8.4',
  62                'OSX10.12',
  63                'Windows_NT',
  64                'Windows_NT_BuildOnly',
  65                'CentOS7.1',
  66                'RHEL7.2',
  67                'Ubuntu16.04',
  68                'Ubuntu16.10',
  69                'Fedora24',
  70                'Tizen']
  71
  72     def static crossList = [
  73                'Ubuntu',
  74                'Debian8.4',
  75                'OSX10.12',
  76                'Windows_NT',
  77                'CentOS7.1',
  78                'RHEL7.2']
  79
  80     // This is a set of JIT stress modes combined with the set of variables that
  81     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  82     // the values are the environment variables
  83     def static jitStressModeScenarios = [
  84                'minopts'                        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
  85                'tieredcompilation'              : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
  86                'no_tiered_compilation'          : ['COMPlus_TieredCompilation' : '0'],
  87                'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
  88                'forcerelocs'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
  89                'jitstress1'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
  90                'jitstress2'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
  91                'jitstress1_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
  92                'jitstress2_tiered'              : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
  93                'jitstressregs1'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
  94                'jitstressregs2'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
  95                'jitstressregs3'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
  96                'jitstressregs4'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
  97                'jitstressregs8'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
  98                'jitstressregs0x10'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
  99                'jitstressregs0x80'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 100                'jitstressregs0x1000'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 101                'jitstress2_jitstressregs1'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
 102                'jitstress2_jitstressregs2'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
 103                'jitstress2_jitstressregs3'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
 104                'jitstress2_jitstressregs4'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
 105                'jitstress2_jitstressregs8'      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
 106                'jitstress2_jitstressregs0x10'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
 107                'jitstress2_jitstressregs0x80'   : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
 108                'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
 109                'tailcallstress'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
 110                'jitsse2only'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
 111                'jitnosimd'                      : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
 112                'jitincompletehwintrinsic'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
 113                'jitx86hwintrinsicnoavx'         : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
 114                'jitx86hwintrinsicnoavx2'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
 115                'jitx86hwintrinsicnosimd'        : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
 116                'jitnox86hwintrinsic'            : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
 117                'corefx_baseline'                : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
 118                'corefx_minopts'                 : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
 119                'corefx_tieredcompilation'       : ['COMPlus_TieredCompilation' : '1'],  // this can be removed once tiered compilation is on by default
 120                'corefx_jitstress1'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
 121                'corefx_jitstress2'              : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
 122                'corefx_jitstressregs1'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
 123                'corefx_jitstressregs2'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
 124                'corefx_jitstressregs3'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
 125                'corefx_jitstressregs4'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
 126                'corefx_jitstressregs8'          : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
 127                'corefx_jitstressregs0x10'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
 128                'corefx_jitstressregs0x80'       : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
 129                'corefx_jitstressregs0x1000'     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
 130                'gcstress0x3'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
 131                'gcstress0xc'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
 132                'zapdisable'                     : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 133                'heapverify1'                    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
 134                'gcstress0xc_zapdisable'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 135                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 136                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 137                'gcstress0xc_jitstress1'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 138                'gcstress0xc_jitstress2'             : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 139                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 140     ]
 141
 142     // This is a set of ReadyToRun stress scenarios
 143     def static r2rStressScenarios = [
 144                'r2r_jitstress1'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
 145                'r2r_jitstress2'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
 146                'r2r_jitstress1_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
 147                'r2r_jitstress2_tiered'      : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
 148                'r2r_jitstressregs1'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
 149                'r2r_jitstressregs2'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
 150                'r2r_jitstressregs3'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
 151                'r2r_jitstressregs4'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
 152                'r2r_jitstressregs8'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
 153                'r2r_jitstressregs0x10'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
 154                'r2r_jitstressregs0x80'      : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
 155                'r2r_jitstressregs0x1000'    : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
 156                'r2r_jitminopts'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
 157                'r2r_jitforcerelocs'         : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
 158                'r2r_gcstress15'             : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
 159                'r2r_no_tiered_compilation'  : ['COMPlus_TieredCompilation' : '0'],
 160     ]
 161
 162     // This is the basic set of scenarios
 163     def static basicScenarios = [
 164                'innerloop',
 165                'normal',
 166                'ilrt',
 167                'r2r',
 168                'longgc',
 169                'formatting',
 170                'gcsimulator',
 171                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 172                'standalone_gc',
 173                'gc_reliability_framework',
 174                'illink',
 175                'corefx_innerloop',
 176                'crossgen_comparison',
 177                'pmi_asm_diffs']
 178
 179     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 180
 181     // Valid PR trigger combinations.
 182     def static prTriggeredValidInnerLoopCombos = [
 183         'Windows_NT': [
 184             'x64': [
 185                 'Checked'
 186             ],
 187             'x86': [
 188                 'Checked',
 189                 'Release'
 190             ],
 191             'arm': [
 192                 'Debug',
 193                 'Checked'
 194             ],
 195             'arm64': [
 196                 'Debug',
 197                 'Checked'
 198             ]
 199         ],
 200         'Windows_NT_BuildOnly': [
 201             'x64': [
 202                 'Checked',
 203                 'Release'
 204             ],
 205             'x86': [
 206                 'Checked',
 207                 'Release'
 208             ],
 209             'arm': [
 210                 'Checked'
 211             ],
 212         ],
 213         'Ubuntu': [
 214             'x64': [
 215                 'Checked'
 216             ],
 217             'arm': [
 218                 'Checked'
 219             ]
 220         ],
 221         'Ubuntu16.04': [
 222             'arm64': [
 223                 'Checked'
 224             ]
 225         ],
 226         'CentOS7.1': [
 227             'x64': [
 228                 'Debug',
 229                 'Checked'
 230             ]
 231         ],
 232         'OSX10.12': [
 233             'x64': [
 234                 'Checked'
 235             ]
 236         ],
 237         'Tizen': [
 238             'armem': [
 239                 'Checked'
 240             ]
 241         ]
 242     ]
 243
 244     // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
 245     // to Tests.lst file categories to exclude.
 246     //
 247     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 248     // and with the same values, with some commented out, for easier maintenance.
 249     //
 250     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 251     //
 252     def static validArmWindowsScenarios = [
 253                'innerloop',
 254                'normal',
 255                // 'ilrt'
 256                'r2r',
 257                // 'longgc'
 258                // 'formatting'
 259                // 'gcsimulator'
 260                // 'jitdiff'
 261                // 'standalone_gc'
 262                // 'gc_reliability_framework'
 263                // 'illink'
 264                // 'corefx_innerloop'
 265                // 'crossgen_comparison'
 266                // 'pmi_asm_diffs'
 267                'r2r_jitstress1',
 268                'r2r_jitstress2',
 269                'r2r_jitstress1_tiered',
 270                'r2r_jitstress2_tiered',
 271                'r2r_jitstressregs1',
 272                'r2r_jitstressregs2',
 273                'r2r_jitstressregs3',
 274                'r2r_jitstressregs4',
 275                'r2r_jitstressregs8',
 276                'r2r_jitstressregs0x10',
 277                'r2r_jitstressregs0x80',
 278                'r2r_jitstressregs0x1000',
 279                'r2r_jitminopts',
 280                'r2r_jitforcerelocs',
 281                'r2r_gcstress15',
 282                'r2r_no_tiered_compilation',
 283                'minopts',
 284                'tieredcompilation',
 285                'no_tiered_compilation',
 286                'no_tiered_compilation_innerloop',
 287                'forcerelocs',
 288                'jitstress1',
 289                'jitstress2',
 290                'jitstress1_tiered',
 291                'jitstress2_tiered',
 292                'jitstressregs1',
 293                'jitstressregs2',
 294                'jitstressregs3',
 295                'jitstressregs4',
 296                'jitstressregs8',
 297                'jitstressregs0x10',
 298                'jitstressregs0x80',
 299                'jitstressregs0x1000',
 300                'jitstress2_jitstressregs1',
 301                'jitstress2_jitstressregs2',
 302                'jitstress2_jitstressregs3',
 303                'jitstress2_jitstressregs4',
 304                'jitstress2_jitstressregs8',
 305                'jitstress2_jitstressregs0x10',
 306                'jitstress2_jitstressregs0x80',
 307                'jitstress2_jitstressregs0x1000',
 308                'tailcallstress',
 309                // 'jitsse2only'                     // Only relevant to xarch
 310                'jitnosimd',                         // Only interesting on platforms where SIMD support exists.
 311                // 'jitincompletehwintrinsic'
 312                // 'jitx86hwintrinsicnoavx'
 313                // 'jitx86hwintrinsicnoavx2'
 314                // 'jitx86hwintrinsicnosimd'
 315                // 'jitnox86hwintrinsic'
 316                'corefx_baseline',                   // corefx tests don't use smarty
 317                'corefx_minopts',                    // corefx tests don't use smarty
 318                'corefx_tieredcompilation',          // corefx tests don't use smarty
 319                'corefx_jitstress1',                 // corefx tests don't use smarty
 320                'corefx_jitstress2',                 // corefx tests don't use smarty
 321                'corefx_jitstressregs1',             // corefx tests don't use smarty
 322                'corefx_jitstressregs2',             // corefx tests don't use smarty
 323                'corefx_jitstressregs3',             // corefx tests don't use smarty
 324                'corefx_jitstressregs4',             // corefx tests don't use smarty
 325                'corefx_jitstressregs8',             // corefx tests don't use smarty
 326                'corefx_jitstressregs0x10',          // corefx tests don't use smarty
 327                'corefx_jitstressregs0x80',          // corefx tests don't use smarty
 328                'corefx_jitstressregs0x1000',        // corefx tests don't use smarty
 329                'gcstress0x3',
 330                'gcstress0xc',
 331                'zapdisable',
 332                'heapverify1',
 333                'gcstress0xc_zapdisable',
 334                'gcstress0xc_zapdisable_jitstress2',
 335                'gcstress0xc_zapdisable_heapverify1',
 336                'gcstress0xc_jitstress1',
 337                'gcstress0xc_jitstress2',
 338                'gcstress0xc_minopts_heapverify1',
 339
 340                //
 341                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 342                //
 343
 344                'minopts_zapdisable',
 345                'gcstress0x3_jitstress1',
 346                'gcstress0x3_jitstress2',
 347                'gcstress0x3_jitstressregs1',
 348                'gcstress0x3_jitstressregs2',
 349                'gcstress0x3_jitstressregs3',
 350                'gcstress0x3_jitstressregs4',
 351                'gcstress0x3_jitstressregs8',
 352                'gcstress0x3_jitstressregs0x10',
 353                'gcstress0x3_jitstressregs0x80',
 354                'gcstress0x3_jitstressregs0x1000',
 355                'gcstress0xc_jitstressregs1',
 356                'gcstress0xc_jitstressregs2',
 357                'gcstress0xc_jitstressregs3',
 358                'gcstress0xc_jitstressregs4',
 359                'gcstress0xc_jitstressregs8',
 360                'gcstress0xc_jitstressregs0x10',
 361                'gcstress0xc_jitstressregs0x80',
 362                'gcstress0xc_jitstressregs0x1000'
 363     ]
 364
 365     def static validLinuxArmScenarios = [
 366                'innerloop',
 367                'normal',
 368                // 'ilrt'
 369                'r2r',
 370                // 'longgc'
 371                // 'formatting'
 372                // 'gcsimulator'
 373                // 'jitdiff'
 374                // 'standalone_gc'
 375                // 'gc_reliability_framework'
 376                // 'illink'
 377                // 'corefx_innerloop'
 378                'crossgen_comparison',
 379                'pmi_asm_diffs',
 380                'r2r_jitstress1',
 381                'r2r_jitstress2',
 382                'r2r_jitstress1_tiered',
 383                'r2r_jitstress2_tiered',
 384                'r2r_jitstressregs1',
 385                'r2r_jitstressregs2',
 386                'r2r_jitstressregs3',
 387                'r2r_jitstressregs4',
 388                'r2r_jitstressregs8',
 389                'r2r_jitstressregs0x10',
 390                'r2r_jitstressregs0x80',
 391                'r2r_jitstressregs0x1000',
 392                'r2r_jitminopts',
 393                'r2r_jitforcerelocs',
 394                'r2r_gcstress15',
 395                'r2r_no_tiered_compilation',
 396                'minopts',
 397                'tieredcompilation',
 398                'no_tiered_compilation',
 399                'no_tiered_compilation_innerloop',
 400                'forcerelocs',
 401                'jitstress1',
 402                'jitstress2',
 403                'jitstress1_tiered',
 404                'jitstress2_tiered',
 405                'jitstressregs1',
 406                'jitstressregs2',
 407                'jitstressregs3',
 408                'jitstressregs4',
 409                'jitstressregs8',
 410                'jitstressregs0x10',
 411                'jitstressregs0x80',
 412                'jitstressregs0x1000',
 413                'jitstress2_jitstressregs1',
 414                'jitstress2_jitstressregs2',
 415                'jitstress2_jitstressregs3',
 416                'jitstress2_jitstressregs4',
 417                'jitstress2_jitstressregs8',
 418                'jitstress2_jitstressregs0x10',
 419                'jitstress2_jitstressregs0x80',
 420                'jitstress2_jitstressregs0x1000',
 421                'tailcallstress',
 422                // 'jitsse2only'                          // Only relevant to xarch
 423                // 'jitnosimd'                            // Only interesting on platforms where SIMD support exists.
 424                // 'jitincompletehwintrinsic'
 425                // 'jitx86hwintrinsicnoavx'
 426                // 'jitx86hwintrinsicnoavx2'
 427                // 'jitx86hwintrinsicnosimd'
 428                // 'jitnox86hwintrinsic'
 429                'corefx_baseline',
 430                'corefx_minopts',
 431                'corefx_tieredcompilation',
 432                'corefx_jitstress1',
 433                'corefx_jitstress2',
 434                'corefx_jitstressregs1',
 435                'corefx_jitstressregs2',
 436                'corefx_jitstressregs3',
 437                'corefx_jitstressregs4',
 438                'corefx_jitstressregs8',
 439                'corefx_jitstressregs0x10',
 440                'corefx_jitstressregs0x80',
 441                'corefx_jitstressregs0x1000',
 442                'gcstress0x3',
 443                'gcstress0xc',
 444                'zapdisable',
 445                'heapverify1',
 446                'gcstress0xc_zapdisable',
 447                'gcstress0xc_zapdisable_jitstress2',
 448                'gcstress0xc_zapdisable_heapverify1',
 449                'gcstress0xc_jitstress1',
 450                'gcstress0xc_jitstress2',
 451                'gcstress0xc_minopts_heapverify1'
 452     ]
 453
 454     def static validLinuxArm64Scenarios = [
 455                'innerloop',
 456                'normal',
 457                // 'ilrt'
 458                'r2r',
 459                // 'longgc'
 460                // 'formatting'
 461                // 'gcsimulator'
 462                // 'jitdiff'
 463                // 'standalone_gc'
 464                // 'gc_reliability_framework'
 465                // 'illink'
 466                // 'corefx_innerloop'
 467                // 'crossgen_comparison'
 468                'pmi_asm_diffs',
 469                'r2r_jitstress1',
 470                'r2r_jitstress2',
 471                'r2r_jitstress1_tiered',
 472                'r2r_jitstress2_tiered',
 473                'r2r_jitstressregs1',
 474                'r2r_jitstressregs2',
 475                'r2r_jitstressregs3',
 476                'r2r_jitstressregs4',
 477                'r2r_jitstressregs8',
 478                'r2r_jitstressregs0x10',
 479                'r2r_jitstressregs0x80',
 480                'r2r_jitstressregs0x1000',
 481                'r2r_jitminopts',
 482                'r2r_jitforcerelocs',
 483                'r2r_gcstress15',
 484                'r2r_no_tiered_compilation',
 485                'minopts',
 486                'tieredcompilation',
 487                'no_tiered_compilation',
 488                'no_tiered_compilation_innerloop',
 489                'forcerelocs',
 490                'jitstress1',
 491                'jitstress2',
 492                'jitstress1_tiered',
 493                'jitstress2_tiered',
 494                'jitstressregs1',
 495                'jitstressregs2',
 496                'jitstressregs3',
 497                'jitstressregs4',
 498                'jitstressregs8',
 499                'jitstressregs0x10',
 500                'jitstressregs0x80',
 501                'jitstressregs0x1000',
 502                'jitstress2_jitstressregs1',
 503                'jitstress2_jitstressregs2',
 504                'jitstress2_jitstressregs3',
 505                'jitstress2_jitstressregs4',
 506                'jitstress2_jitstressregs8',
 507                'jitstress2_jitstressregs0x10',
 508                'jitstress2_jitstressregs0x80',
 509                'jitstress2_jitstressregs0x1000',
 510                'tailcallstress',
 511                // 'jitsse2only'                         // Only relevant to xarch
 512                'jitnosimd',                             // Only interesting on platforms where SIMD support exists.
 513                // 'jitincompletehwintrinsic'
 514                // 'jitx86hwintrinsicnoavx'
 515                // 'jitx86hwintrinsicnoavx2'
 516                // 'jitx86hwintrinsicnosimd'
 517                // 'jitnox86hwintrinsic'
 518                'corefx_baseline',
 519                'corefx_minopts',
 520                'corefx_tieredcompilation',
 521                'corefx_jitstress1',
 522                'corefx_jitstress2',
 523                'corefx_jitstressregs1',
 524                'corefx_jitstressregs2',
 525                'corefx_jitstressregs3',
 526                'corefx_jitstressregs4',
 527                'corefx_jitstressregs8',
 528                'corefx_jitstressregs0x10',
 529                'corefx_jitstressregs0x80',
 530                'corefx_jitstressregs0x1000',
 531                'gcstress0x3',
 532                'gcstress0xc',
 533                'zapdisable',
 534                'heapverify1',
 535                'gcstress0xc_zapdisable',
 536                'gcstress0xc_zapdisable_jitstress2',
 537                'gcstress0xc_zapdisable_heapverify1',
 538                'gcstress0xc_jitstress1',
 539                'gcstress0xc_jitstress2',
 540                'gcstress0xc_minopts_heapverify1'
 541     ]
 542
 543     def static configurationList = ['Debug', 'Checked', 'Release']
 544
 545     // This is the set of architectures
 546     // Some of these are pseudo-architectures:
 547     //    armem -- ARM builds/runs using an emulator. Used for Tizen runs.
 548     //    x86_arm_altjit -- ARM runs on x86 using the ARM altjit
 549     //    x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
 550     def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 551
 552     // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
 553     def static armWindowsCrossArchitectureList = ['arm', 'arm64']
 554 }
 555
 556 // **************************************************************
 557 // Create some specific views
 558 //
 559 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 560 // views based on a single regular expression. These views will be generated by adding a
 561 // specific set of jobs to them.
 562 //
 563 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 564 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 565 // those are required, don't add them (which simplifies the view pages, as well).
 566 // **************************************************************
 567
 568 class Views {
 569     def static MergeJobView = null
 570     def static PeriodicJobView = null
 571     def static ArchitectureViews = [:]
 572     def static OSViews = [:]
 573 }
 574
 575 // MergeJobView: include all jobs that execute when a PR change is merged.
 576 Views.MergeJobView = listView('Merge') {
 577     recurse()
 578     columns {
 579         status()
 580         weather()
 581         name()
 582         lastSuccess()
 583         lastFailure()
 584         lastDuration()
 585         buildButton()
 586     }
 587 }
 588
 589 // PeriodicJobView: include all jobs that execute on a schedule
 590 Views.PeriodicJobView = listView('Periodic') {
 591     recurse()
 592     columns {
 593         status()
 594         weather()
 595         name()
 596         lastSuccess()
 597         lastFailure()
 598         lastDuration()
 599         buildButton()
 600     }
 601 }
 602
 603 // Create a view for non-PR jobs for each architecture.
 604 Constants.architectureList.each { architecture ->
 605     Views.ArchitectureViews[architecture] = listView(architecture) {
 606         recurse()
 607         columns {
 608             status()
 609             weather()
 610             name()
 611             lastSuccess()
 612             lastFailure()
 613             lastDuration()
 614             buildButton()
 615         }
 616     }
 617 }
 618
 619 // Create a view for non-PR jobs for each OS.
 620 Constants.osList.each { os ->
 621     // Don't create one for the special 'Windows_NT_BuildOnly'
 622     if (os == 'Windows_NT_BuildOnly') {
 623         return
 624     }
 625     Views.OSViews[os] = listView(os) {
 626         recurse()
 627         columns {
 628             status()
 629             weather()
 630             name()
 631             lastSuccess()
 632             lastFailure()
 633             lastDuration()
 634             buildButton()
 635         }
 636     }
 637 }
 638
 639 def static addToMergeView(def job) {
 640     Views.MergeJobView.with {
 641         jobs {
 642             name(job.name)
 643         }
 644     }
 645 }
 646
 647 def static addToPeriodicView(def job) {
 648     Views.PeriodicJobView.with {
 649         jobs {
 650             name(job.name)
 651         }
 652     }
 653 }
 654
 655 def static addToViews(def job, def isFlowJob, def isPR, def architecture, def os, def configuration, def scenario) {
 656     if (isPR) {
 657         // No views want PR jobs currently.
 658         return
 659     }
 660
 661     // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
 662     // "top-level" jobs. Build only jobs are such jobs.
 663     if (os == 'Windows_NT_BuildOnly') {
 664         return
 665     }
 666
 667     if (!isFlowJob) {
 668         // For non-flow jobs, which ones are only used by flow jobs?
 669         if ((architecture == 'arm') || (architecture == 'arm64')) {
 670             if (isCoreFxScenario(scenario)) {
 671                 // We have corefx-specific scenario builds for each of the runs, but these are driven by flow jobs.
 672                 return
 673             }
 674
 675             // We're left with the basic normal/innerloop builds. We might want to remove these from the views also, if desired.
 676             // However, there are a few, like the Debug Build, that is build only, not "Build and Test", that we should leave.
 677         }
 678     }
 679
 680     // Add to architecture view.
 681     Views.ArchitectureViews[architecture].with {
 682         jobs {
 683             name(job.name)
 684         }
 685     }
 686
 687     // Add to OS view.
 688     Views.OSViews[os].with {
 689         jobs {
 690             name(job.name)
 691         }
 692     }
 693 }
 694
 695 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 696     addToPeriodicView(job)
 697     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 698 }
 699
 700 def static addGithubPushTriggerHelper(def job) {
 701     addToMergeView(job)
 702     Utilities.addGithubPushTrigger(job)
 703 }
 704
 705
 706 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 707     assert os instanceof String
 708     assert architecture instanceof String
 709
 710     def armArches = ['arm', 'armem', 'arm64']
 711
 712     if (!(architecture in armArches)) {
 713         assert options == null
 714         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 715
 716         return
 717     }
 718
 719     // This is an arm(64) job.
 720     //
 721     // There are several options.
 722     //
 723     // Windows_NT
 724     //
 725     // Arm32 (Build) -> latest-arm64
 726     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
 727     // Arm32 (Test)  -> arm64-windows_nt
 728     //       |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
 729     //
 730     // Arm64 (Build) -> latest-arm64
 731     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 732     // Arm64 (Test)  -> arm64-windows_nt
 733     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 734     //
 735     // Ubuntu
 736     //
 737     // Arm32 emulator (Build, Test) -> arm-cross-latest
 738     //       |-> os == "Tizen" && (architecture == "armem")
 739     //
 740     // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
 741     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
 742     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
 743     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
 744     // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
 745     //       |-> os == "Ubuntu" && (architecture == "arm")
 746     //
 747     // Arm64 (Build) -> arm64-cross-latest
 748     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
 749     // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
 750     //       |-> os != "Windows_NT" && architecture == "arm64"
 751     //
 752     // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
 753     // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
 754     //
 755     // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
 756     // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 757
 758     // This has to be a arm arch
 759     assert architecture in armArches
 760     if (os == "Windows_NT") {
 761         // arm32/arm64 Windows jobs share the same machines for now
 762         def isBuild = options['use_arm64_build_machine'] == true
 763
 764         if (isBuild == true) {
 765             job.with {
 766                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
 767             }
 768         } else {
 769             Utilities.setMachineAffinity(job, 'windows.10.arm64.open')
 770         }
 771     } else {
 772         assert os != 'Windows_NT'
 773
 774         if (architecture == 'armem') {
 775             // arm emulator (Tizen). Build and test on same machine,
 776             // using Docker.
 777             assert os == 'Tizen'
 778             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 779         }
 780         else {
 781             // arm/arm64 Ubuntu on hardware.
 782             assert architecture == 'arm' || architecture == 'arm64'
 783             def isFlow  = (options != null) && (options['is_flow_job'] == true)
 784             def isBuild = (options != null) && (options['is_build_job'] == true)
 785             if (isFlow || isBuild) {
 786                 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
 787                 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
 788                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
 789             } else {
 790                 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
 791                 if (architecture == 'arm64') {
 792                     assert os == 'Ubuntu16.04'
 793                     job.with {
 794                         label('Ubuntu.1604.Arm64.Open')
 795                     }
 796                 }
 797                 else {
 798                     assert os == 'Ubuntu'
 799                     job.with {
 800                         label('ubuntu.1404.arm32.open')
 801                     }
 802                 }
 803             }
 804         }
 805     }
 806 }
 807
 808 // setJobMachineAffinity: compute the machine affinity options for a job,
 809 // then set the job with those affinity options.
 810 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
 811 {
 812     assert (isBuildJob  && !isTestJob && !isFlowJob) ||
 813            (!isBuildJob && isTestJob  && !isFlowJob) ||
 814            (!isBuildJob && !isTestJob && isFlowJob)
 815
 816     def affinityOptions = null
 817     def affinityArchitecture = architecture
 818
 819     if (os == "Windows_NT") {
 820         if (architecture in Constants.armWindowsCrossArchitectureList) {
 821             if (isBuildJob) {
 822                 affinityOptions = [ "use_arm64_build_machine" : true ]
 823             } else if (isTestJob) {
 824                 affinityOptions = [ "use_arm64_build_machine" : false ]
 825             } else if (isFlowJob) {
 826                 // For the flow jobs set the machine affinity as x64
 827                 affinityArchitecture = 'x64'
 828             }
 829         }
 830     }
 831     else {
 832         if ((architecture == 'arm64') || (architecture == 'arm')) {
 833             if (isBuildJob) {
 834                 affinityOptions = ['is_build_job': true]
 835             } else if (isFlowJob) {
 836                 affinityOptions = ['is_flow_job': true]
 837             }
 838         }
 839     }
 840
 841     setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
 842 }
 843
 844 def static isGCStressRelatedTesting(def scenario) {
 845     // The 'r2r_gcstress15' scenario is a basic scenario.
 846     // Detect it and make it a GCStress related.
 847     if (scenario == 'r2r_gcstress15')
 848     {
 849         return true;
 850     }
 851
 852     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 853     def scenarioName = scenario.toLowerCase()
 854     def isGCStressTesting = false
 855     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 856         if (k in gcStressTestEnvVars) {
 857             isGCStressTesting = true;
 858         }
 859     }
 860     return isGCStressTesting
 861 }
 862
 863 def static isCoreFxScenario(def scenario) {
 864     def corefx_prefix = 'corefx_'
 865     if (scenario.length() < corefx_prefix.length()) {
 866         return false
 867     }
 868     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 869 }
 870
 871 def static isR2RBaselineScenario(def scenario) {
 872     return (scenario == 'r2r')
 873 }
 874
 875 def static isR2RStressScenario(def scenario) {
 876     return Constants.r2rStressScenarios.containsKey(scenario)
 877 }
 878
 879 def static isR2RScenario(def scenario) {
 880     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 881 }
 882
 883 def static isJitStressScenario(def scenario) {
 884     return Constants.jitStressModeScenarios.containsKey(scenario)
 885 }
 886
 887 def static isLongGc(def scenario) {
 888     return (scenario == 'longgc' || scenario == 'gcsimulator')
 889 }
 890
 891 def static isJitDiff(def scenario) {
 892     return (scenario == 'jitdiff')
 893 }
 894
 895 def static isGcReliabilityFramework(def scenario) {
 896     return (scenario == 'gc_reliability_framework')
 897 }
 898
 899 def static isArmWindowsScenario(def scenario) {
 900     return Constants.validArmWindowsScenarios.contains(scenario)
 901 }
 902
 903 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
 904     if (isBuildOnly == true) {
 905         os = 'Windows_NT_BuildOnly'
 906     }
 907
 908     def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
 909     if (validOsPrTriggerArchConfigs != null) {
 910         def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
 911         if (validOsPrTriggerConfigs != null) {
 912             if (configuration in validOsPrTriggerConfigs) {
 913                 return true
 914             }
 915         }
 916     }
 917
 918     return false
 919 }
 920
 921 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
 922 // scheduled with a default PR trigger despite the correlation being true at the moment.
 923 def static isInnerloopTestScenario(def scenario) {
 924     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 925 }
 926
 927 def static isCrossGenComparisonScenario(def scenario) {
 928     return (scenario == 'crossgen_comparison')
 929 }
 930
 931 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
 932     assert isCrossGenComparisonScenario(scenario)
 933     return (os == 'Ubuntu' && architecture == 'arm' && (configuration == 'Checked' || configuration == 'Release'))
 934 }
 935
 936 def static getFxBranch(def branch) {
 937     def fxBranch = branch
 938     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
 939     // branch even though CoreFX doesn't have such a branch.
 940     if (branch == 'dev/unix_test_workflow') {
 941         fxBranch = 'master'
 942     }
 943     return fxBranch
 944 }
 945
 946 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 947     // 2 hours (120 minutes) is the default timeout
 948     def timeout = 120
 949
 950     if (!isInnerloopTestScenario(scenario)) {
 951         // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 952         timeout = 240
 953     }
 954
 955     if (!isBuildOnly) {
 956         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 957         if (isGCStressRelatedTesting(scenario)) {
 958             timeout = 4320
 959         }
 960         else if (isCoreFxScenario(scenario)) {
 961             timeout = 360
 962         }
 963         else if (isJitStressScenario(scenario)) {
 964             timeout = 300
 965         }
 966         else if (isR2RBaselineScenario(scenario)) {
 967             timeout = 240
 968         }
 969         else if (isLongGc(scenario)) {
 970             timeout = 1440
 971         }
 972         else if (isJitDiff(scenario)) {
 973             timeout = 240
 974         }
 975         else if (isGcReliabilityFramework(scenario)) {
 976             timeout = 1440
 977         }
 978         else if (architecture == 'armem' || architecture == 'arm64') {
 979             timeout = 240
 980         }
 981
 982         if (architecture == 'arm') {
 983             // ARM32 machines are particularly slow.
 984             timeout += 120
 985         }
 986     }
 987
 988     if (configuration == 'Debug') {
 989         // Debug runs can be very slow. Add an hour.
 990         timeout += 60
 991     }
 992
 993     if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
 994         // AltJit runs compile all methods twice.
 995         timeout *= 2
 996     }
 997
 998     // If we've changed the timeout from the default, set it in the job.
 999
1000     if (timeout != 120) {
1001         Utilities.setJobTimeout(newJob, timeout)
1002     }
1003 }
1004
1005 def static getJobFolder(def scenario) {
1006     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1007         return 'jitstress'
1008     }
1009     if (scenario == 'illink') {
1010         return 'illink'
1011     }
1012     return ''
1013 }
1014
1015 def static getStressModeDisplayName(def scenario) {
1016     def displayStr = ''
1017     Constants.jitStressModeScenarios[scenario].each{ k, v ->
1018         def prefixLength = 'COMPlus_'.length()
1019         if (k.length() >= prefixLength) {
1020             def modeName = k.substring(prefixLength, k.length())
1021             if (displayStr != '') {
1022                 // Separate multiple variables with a space.
1023                 displayStr += ' '
1024             }
1025             displayStr += modeName + '=' + v
1026         }
1027     }
1028
1029     if (isCoreFxScenario(scenario)) {
1030         displayStr = ('CoreFx ' + displayStr).trim()
1031     }
1032
1033     return displayStr
1034 }
1035
1036 def static getR2RDisplayName(def scenario) {
1037     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1038     def displayStr = scenario
1039     def prefixLength = 'r2r_'.length()
1040     if (displayStr.length() >= prefixLength) {
1041         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1042     } else if (scenario == 'r2r') {
1043         displayStr = "R2R"
1044     }
1045     return displayStr
1046 }
1047
1048 def static getScenarioDisplayString(def scenario) {
1049     switch (scenario) {
1050         case 'innerloop':
1051             return "Innerloop Build and Test"
1052
1053         case 'no_tiered_compilation_innerloop':
1054             def displayStr = getStressModeDisplayName(scenario)
1055             return "Innerloop Build and Test (Jit - ${displayStr})"
1056
1057         case 'corefx_innerloop':
1058             return "CoreFX Tests"
1059
1060         case 'normal':
1061             return "Build and Test"
1062
1063         case 'jitdiff':
1064             return "Jit Diff Build and Test"
1065
1066         case 'ilrt':
1067             return "IL RoundTrip Build and Test"
1068
1069         case 'longgc':
1070             return "Long-Running GC Build & Test"
1071
1072         case 'gcsimulator':
1073             return "GC Simulator"
1074
1075         case 'standalone_gc':
1076             return "Standalone GC"
1077
1078         case 'gc_reliability_framework':
1079             return "GC Reliability Framework"
1080
1081         case 'illink':
1082             return "via ILLink"
1083
1084         default:
1085             if (isJitStressScenario(scenario)) {
1086                 def displayStr = getStressModeDisplayName(scenario)
1087                 return "Build and Test (Jit - ${displayStr})"
1088             }
1089             else if (isR2RScenario(scenario)) {
1090                 def displayStr = getR2RDisplayName(scenario)
1091                 return "${displayStr} Build and Test"
1092             }
1093             else {
1094                 return "${scenario}"
1095             }
1096             break
1097     }
1098
1099     println("Unknown scenario: ${scenario}");
1100     assert false
1101 }
1102
1103 //
1104 // Functions to create an environment script.
1105 //      envScriptCreate -- initialize the script (call first)
1106 //      envScriptFinalize -- finalize the script (call last)
1107 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
1108 //      envScriptAppendExistingScript -- append an existing script to the generated script
1109 //
1110 // Each script returns a string of commands. Concatenate all the strings together before
1111 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1112 //
1113
1114 // Initialize the environment setting script.
1115 def static envScriptCreate(def os, def stepScriptLocation) {
1116     def stepScript = ''
1117     if (os == 'Windows_NT') {
1118         stepScript += "echo Creating TestEnv script\r\n"
1119         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1120
1121         // Create at least an empty script.
1122         stepScript += "echo. > ${stepScriptLocation}\r\n"
1123     }
1124     else {
1125         stepScript += "echo Creating environment setting script\n"
1126         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1127     }
1128
1129     return stepScript
1130 }
1131
1132 // Generates the string for setting stress mode variables.
1133 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1134     def stepScript = ''
1135     if (os == 'Windows_NT') {
1136         stressModeVars.each{ k, v ->
1137             // Write out what we are writing to the script file
1138             stepScript += "echo Setting ${k}=${v}\r\n"
1139             // Write out the set itself to the script file`
1140             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1141         }
1142     }
1143     else {
1144         stressModeVars.each{ k, v ->
1145             // Write out what we are writing to the script file
1146             stepScript += "echo Setting ${k}=${v}\n"
1147             // Write out the set itself to the script file`
1148             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1149         }
1150     }
1151
1152     return stepScript
1153 }
1154
1155 // Append an existing script to an environment script.
1156 // Returns string of commands to do this.
1157 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1158     assert (os == 'Windows_NT')
1159     def stepScript = ''
1160
1161     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1162     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1163
1164     return stepScript
1165 }
1166
1167 // Finalize an environment setting script.
1168 // Returns string of commands to do this.
1169 def static envScriptFinalize(def os, def stepScriptLocation) {
1170     def stepScript = ''
1171
1172     if (os == 'Windows_NT') {
1173         // Display the resulting script. This is useful when looking at the output log file.
1174         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1175         stepScript += "type ${stepScriptLocation}\r\n"
1176     }
1177     else {
1178         stepScript += "chmod +x ${stepScriptLocation}\n"
1179     }
1180
1181     return stepScript
1182 }
1183
1184 def static isNeedDocker(def architecture, def os, def isBuild) {
1185     if (isBuild) {
1186         if (architecture == 'x86' && os == 'Ubuntu') {
1187             return true
1188         }
1189         else if (architecture == 'armem') {
1190             return true
1191         }
1192         else if (architecture == 'arm') {
1193             if (os == 'Ubuntu') {
1194                 return true
1195             }
1196         }
1197         else if (architecture == 'arm64') {
1198             if (os == 'Ubuntu16.04') {
1199                 return true
1200             }
1201         }
1202     }
1203     else {
1204         if (architecture == 'x86' && os == 'Ubuntu') {
1205             return true
1206         }
1207     }
1208     return false
1209 }
1210
1211 def static getDockerImageName(def architecture, def os, def isBuild) {
1212     // We must change some docker private images to official later
1213     if (isBuild) {
1214         if (architecture == 'x86' && os == 'Ubuntu') {
1215             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1216         }
1217         else if (architecture == 'armem') {
1218             if (os == 'Tizen') {
1219                 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1220             }
1221         }
1222         else if (architecture == 'arm') {
1223             if (os == 'Ubuntu') {
1224                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1225             }
1226         }
1227         else if (architecture == 'arm64') {
1228             if (os == 'Ubuntu16.04') {
1229                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1230             }
1231         }
1232     }
1233     else {
1234         if (architecture == 'x86' && os == 'Ubuntu') {
1235             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1236         }
1237     }
1238     println("Unknown architecture to use docker: ${architecture} ${os}");
1239     assert false
1240 }
1241
1242
1243 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1244 // and only allowing using this hardware in some specific branches.
1245 def static jobRequiresLimitedHardware(def architecture, def os) {
1246     if (architecture == 'arm') {
1247         // arm Windows and Linux hardware is limited.
1248         return true
1249     }
1250     else if (architecture == 'arm64') {
1251         // arm64 Windows and Linux hardware is limited.
1252         return true
1253     }
1254     else {
1255         return false
1256     }
1257 }
1258
1259 // Calculates the name of the build job based on some typical parameters.
1260 //
1261 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1262     // If the architecture is x64, do not add that info into the build name.
1263     // Need to change around some systems and other builds to pick up the right builds
1264     // to do that.
1265
1266     def suffix = scenario != 'normal' ? "_${scenario}" : '';
1267     if (isBuildOnly) {
1268         suffix += '_bld'
1269     }
1270     def baseName = ''
1271     switch (architecture) {
1272         case 'x64':
1273             if (scenario == 'normal') {
1274                 // For now we leave x64 off of the name for compatibility with other jobs
1275                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1276             }
1277             else if (scenario == 'formatting') {
1278                 // we don't care about the configuration for the formatting job. It runs all configs
1279                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1280             }
1281             else {
1282                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1283             }
1284             break
1285         case 'armem':
1286             // These are cross builds
1287             assert os == 'Tizen'
1288             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1289             break
1290         case 'arm':
1291         case 'arm64':
1292             // These are cross builds
1293             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1294             break
1295         case 'x86':
1296         case 'x86_arm_altjit':
1297         case 'x64_arm64_altjit':
1298             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1299             break
1300         default:
1301             println("Unknown architecture: ${architecture}");
1302             assert false
1303             break
1304     }
1305
1306     return baseName + suffix
1307 }
1308
1309 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1310     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1311
1312     // Limited hardware is restricted for non-PR triggers to certain branches.
1313     if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1314         return
1315     }
1316
1317     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1318     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1319     if (architecture == 'x86' && os == 'Ubuntu') {
1320         return
1321     }
1322
1323     // Check scenario.
1324     switch (scenario) {
1325         case 'crossgen_comparison':
1326             if (isFlowJob && os == 'Ubuntu' && architecture == 'arm' && (configuration == 'Checked' || configuration == 'Release')) {
1327                 addPeriodicTriggerHelper(job, '@daily')
1328             }
1329             break
1330
1331         case 'pmi_asm_diffs':
1332             // No non-PR triggers for now.
1333             break
1334
1335         case 'normal':
1336             switch (architecture) {
1337                 case 'x64':
1338                 case 'x86':
1339                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1340                         addPeriodicTriggerHelper(job, '@daily')
1341                     }
1342                     else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1343                         addGithubPushTriggerHelper(job)
1344                     }
1345                     break
1346                 case 'arm64':
1347                     if (os == 'Windows_NT') {
1348                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1349                             // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1350                             // Do the builds daily.
1351                             addPeriodicTriggerHelper(job, '@daily')
1352                         }
1353                     }
1354                     else {
1355                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1356                         if (isFlowJob) {
1357                             addPeriodicTriggerHelper(job, '@daily')
1358                         }
1359                     }
1360                     break
1361                 case 'arm':
1362                     if (os == 'Windows_NT') {
1363                         if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1364                             // We would normally want a push trigger, but with limited hardware we can't keep up.
1365                             // Do the builds daily.
1366                             addPeriodicTriggerHelper(job, '@daily')
1367                         }
1368                     }
1369                     else {
1370                         assert os == 'Ubuntu'
1371                         // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1372                         if (isFlowJob) {
1373                             // Currently no push triggers, with limited arm Linux hardware.
1374                             // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1375                             addPeriodicTriggerHelper(job, '@daily')
1376                         }
1377                     }
1378                     break
1379                 case 'armem':
1380                     addGithubPushTriggerHelper(job)
1381                     break
1382                 case 'x86_arm_altjit':
1383                 case 'x64_arm64_altjit':
1384                     // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1385                     if (configuration == 'Checked') {
1386                         addGithubPushTriggerHelper(job)
1387                     }
1388                     break
1389                 default:
1390                     println("Unknown architecture: ${architecture}");
1391                     assert false
1392                     break
1393             }
1394             break
1395         case 'r2r':
1396             assert !(os in bidailyCrossList)
1397             // r2r gets a push trigger for checked/release
1398             if (configuration == 'Checked' || configuration == 'Release') {
1399                 if (architecture == 'x64' && os != 'OSX10.12') {
1400                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1401                     if (isFlowJob || os == 'Windows_NT') {
1402                         addGithubPushTriggerHelper(job)
1403                     }
1404                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1405                 } else if (architecture == 'x64' && os == 'OSX10.12'){
1406                     if (isFlowJob) {
1407                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
1408                     }
1409                 }
1410                 // For x86, only add per-commit jobs for Windows
1411                 else if (architecture == 'x86') {
1412                     if (os == 'Windows_NT') {
1413                         addGithubPushTriggerHelper(job)
1414                     }
1415                 }
1416                 // arm r2r jobs should only run weekly.
1417                 else if (architecture == 'arm') {
1418                     if (isFlowJob) {
1419                         addPeriodicTriggerHelper(job, '@weekly')
1420                     }
1421                 }
1422                 // arm64 r2r jobs should only run weekly.
1423                 else if (architecture == 'arm64') {
1424                     if (isFlowJob) {
1425                         addPeriodicTriggerHelper(job, '@weekly')
1426                     }
1427                 }
1428             }
1429             break
1430         case 'r2r_jitstress1':
1431         case 'r2r_jitstress2':
1432         case 'r2r_jitstress1_tiered':
1433         case 'r2r_jitstress2_tiered':
1434         case 'r2r_jitstressregs1':
1435         case 'r2r_jitstressregs2':
1436         case 'r2r_jitstressregs3':
1437         case 'r2r_jitstressregs4':
1438         case 'r2r_jitstressregs8':
1439         case 'r2r_jitstressregs0x10':
1440         case 'r2r_jitstressregs0x80':
1441         case 'r2r_jitstressregs0x1000':
1442         case 'r2r_jitminopts':
1443         case 'r2r_jitforcerelocs':
1444         case 'r2r_gcstress15':
1445         case 'r2r_no_tiered_compilation':
1446             assert !(os in bidailyCrossList)
1447
1448             // GCStress=C is currently not supported on OS X
1449             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1450                 break
1451             }
1452
1453             if (configuration == 'Checked' || configuration == 'Release') {
1454                 if (architecture == 'x64') {
1455                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1456                     if (isFlowJob || os == 'Windows_NT') {
1457                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1458                     }
1459                 }
1460                 // For x86, only add periodic jobs for Windows
1461                 else if (architecture == 'x86') {
1462                     if (os == 'Windows_NT') {
1463                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1464                     }
1465                 }
1466                 else if (architecture == 'arm') {
1467                     if (isFlowJob) {
1468                         addPeriodicTriggerHelper(job, '@weekly')
1469                     }
1470                 }
1471                 else if (architecture == 'arm64') {
1472                     if (isFlowJob) {
1473                         addPeriodicTriggerHelper(job, '@weekly')
1474                     }
1475                 }
1476             }
1477             break
1478         case 'longgc':
1479             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1480             assert configuration == 'Release'
1481             assert architecture == 'x64'
1482             addPeriodicTriggerHelper(job, '@daily')
1483             // TODO: Add once external email sending is available again
1484             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1485             break
1486         case 'gcsimulator':
1487             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1488             assert configuration == 'Release'
1489             assert architecture == 'x64'
1490             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1491             // TODO: Add once external email sending is available again
1492             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1493             break
1494         case 'standalone_gc':
1495             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1496             assert (configuration == 'Release' || configuration == 'Checked')
1497             // TODO: Add once external email sending is available again
1498             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1499             addPeriodicTriggerHelper(job, '@daily')
1500             break
1501         case 'gc_reliability_framework':
1502             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1503             assert (configuration == 'Release' || configuration == 'Checked')
1504             // Only triggered by phrase.
1505             break
1506         case 'ilrt':
1507             assert !(os in bidailyCrossList)
1508             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1509             if (architecture == 'x64' && configuration == 'Release') {
1510                 if (isFlowJob || os == 'Windows_NT') {
1511                     addPeriodicTriggerHelper(job, '@daily')
1512                 }
1513             }
1514             break
1515         case 'jitdiff':
1516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1517             assert configuration == 'Checked'
1518             assert (architecture == 'x64' || architecture == 'x86')
1519             addGithubPushTriggerHelper(job)
1520             break
1521         case 'formatting':
1522             assert (os == 'Windows_NT' || os == "Ubuntu")
1523             assert architecture == 'x64'
1524             addGithubPushTriggerHelper(job)
1525             break
1526         case 'jitstressregs1':
1527         case 'jitstressregs2':
1528         case 'jitstressregs3':
1529         case 'jitstressregs4':
1530         case 'jitstressregs8':
1531         case 'jitstressregs0x10':
1532         case 'jitstressregs0x80':
1533         case 'jitstressregs0x1000':
1534         case 'minopts':
1535         case 'tieredcompilation':
1536         case 'no_tiered_compilation':
1537         case 'forcerelocs':
1538         case 'jitstress1':
1539         case 'jitstress2':
1540         case 'jitstress1_tiered':
1541         case 'jitstress2_tiered':
1542         case 'jitstress2_jitstressregs1':
1543         case 'jitstress2_jitstressregs2':
1544         case 'jitstress2_jitstressregs3':
1545         case 'jitstress2_jitstressregs4':
1546         case 'jitstress2_jitstressregs8':
1547         case 'jitstress2_jitstressregs0x10':
1548         case 'jitstress2_jitstressregs0x80':
1549         case 'jitstress2_jitstressregs0x1000':
1550         case 'tailcallstress':
1551         case 'jitsse2only':
1552         case 'jitnosimd':
1553         case 'jitnox86hwintrinsic':
1554         case 'jitincompletehwintrinsic':
1555         case 'jitx86hwintrinsicnoavx':
1556         case 'jitx86hwintrinsicnoavx2':
1557         case 'jitx86hwintrinsicnosimd':
1558         case 'corefx_baseline':
1559         case 'corefx_minopts':
1560         case 'corefx_tieredcompilation':
1561         case 'corefx_jitstress1':
1562         case 'corefx_jitstress2':
1563         case 'corefx_jitstressregs1':
1564         case 'corefx_jitstressregs2':
1565         case 'corefx_jitstressregs3':
1566         case 'corefx_jitstressregs4':
1567         case 'corefx_jitstressregs8':
1568         case 'corefx_jitstressregs0x10':
1569         case 'corefx_jitstressregs0x80':
1570         case 'corefx_jitstressregs0x1000':
1571         case 'zapdisable':
1572             if (os == 'CentOS7.1') {
1573                 break
1574             }
1575             if (os in bidailyCrossList) {
1576                 break
1577             }
1578             // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1579             // build for the flow job. We don't need cron jobs for these. Note that the
1580             // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1581             // function very early, so only non-Windows gets here.
1582             if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1583                 break
1584             }
1585             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1586                 break
1587             }
1588             if (jobRequiresLimitedHardware(architecture, os)) {
1589                 if ((architecture == 'arm64') && (os == 'Ubuntu16.04')) {
1590                     // These jobs are very fast on Linux/arm64 hardware, so run them daily.
1591                     addPeriodicTriggerHelper(job, '@daily')
1592                 }
1593                 else {
1594                     addPeriodicTriggerHelper(job, '@weekly')
1595                 }
1596             }
1597             else {
1598                 addPeriodicTriggerHelper(job, '@daily')
1599             }
1600             break
1601         case 'heapverify1':
1602         case 'gcstress0x3':
1603             if (os == 'CentOS7.1') {
1604                 break
1605             }
1606             if (os in bidailyCrossList) {
1607                 break
1608             }
1609             addPeriodicTriggerHelper(job, '@weekly')
1610             break
1611         case 'gcstress0xc':
1612         case 'gcstress0xc_zapdisable':
1613         case 'gcstress0xc_zapdisable_jitstress2':
1614         case 'gcstress0xc_zapdisable_heapverify1':
1615         case 'gcstress0xc_jitstress1':
1616         case 'gcstress0xc_jitstress2':
1617         case 'gcstress0xc_minopts_heapverify1':
1618             if (os == 'OSX10.12') {
1619                 // GCStress=C is currently not supported on OS X
1620                 break
1621             }
1622             if (os == 'CentOS7.1') {
1623                 break
1624             }
1625             if (os in bidailyCrossList) {
1626                 break
1627             }
1628             addPeriodicTriggerHelper(job, '@weekly')
1629             break
1630
1631         case 'illink':
1632             // Testing on other operating systems TBD
1633             assert (os == 'Windows_NT' || os == 'Ubuntu')
1634             if (architecture == 'x64' || architecture == 'x86') {
1635                 if (configuration == 'Checked') {
1636                     addPeriodicTriggerHelper(job, '@daily')
1637                 }
1638             }
1639             break
1640
1641         default:
1642             println("Unknown scenario: ${scenario}");
1643             assert false
1644             break
1645     }
1646     return
1647 }
1648
1649 // **************************
1650 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1651 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1652 // builds will do a couple extra steps.
1653 // **************************
1654
1655 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1656 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1657 // it's just used for internal builds
1658 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1659 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1660     def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1661
1662     if (isWindowsBuildOnlyJob) {
1663         return
1664     }
1665
1666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1667     // Non pull request builds.
1668     if (!isPR) {
1669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1670         return
1671     }
1672
1673     def arm64Users = [
1674         'adityamandaleeka',
1675         'AndyAyersMS',
1676         'briansull',
1677         'BruceForstall',
1678         'CarolEidt',
1679         'davidwrighton',
1680         'echesakovMSFT',
1681         'erozenfeld',
1682         'janvorli',
1683         'jashook',
1684         'pgodeq',
1685         'RussKeldorph',
1686         'sandreenko',
1687         'sdmaclea',
1688         'swaroop-sridhar',
1689         'jkotas',
1690         'markwilkie',
1691         'weshaggard'
1692     ]
1693
1694     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1695     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1696
1697     def contextString = ""
1698     def triggerString = ""
1699     def needsTrigger = true
1700     def isDefaultTrigger = false
1701     def isArm64PrivateJob = false
1702     def scenarioString = ""
1703
1704     // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1705     // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1706     // some jobs where the Debug configuration just does builds, no tests.
1707     //
1708     // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1709     // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1710     // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1711     // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1712
1713     if (architecture == 'armem') {
1714         assert os == 'Tizen'
1715         architecture = 'armel'
1716     }
1717
1718     switch (architecture) {
1719         case 'x64_arm64_altjit':
1720         case 'x86_arm_altjit':
1721             // TODO: for consistency, add "Build and Test" at end.
1722             contextString = "${os} ${architecture} ${configuration} ${scenario}"
1723             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1724             break
1725
1726         case 'armel':
1727         case 'arm':
1728         case 'arm64':
1729             contextString = "${os} ${architecture} Cross ${configuration}"
1730             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1731
1732             if (scenario == 'innerloop') {
1733                 contextString += " Innerloop"
1734                 triggerString += "\\W+Innerloop"
1735             }
1736             else {
1737                 contextString += " ${scenario}"
1738                 triggerString += "\\W+${scenario}"
1739             }
1740
1741             if (scenario == 'pmi_asm_diffs') {
1742                 // Don't add the "Build and Test" part
1743             }
1744             else if (configuration == 'Debug') {
1745                 contextString += " Build"
1746                 triggerString += "\\W+Build"
1747             }
1748             else {
1749                 contextString += " Build and Test"
1750                 triggerString += "\\W+Build and Test"
1751             }
1752
1753             triggerString += ".*"
1754             break
1755
1756         default:
1757             scenarioString = getScenarioDisplayString(scenario)
1758             contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1759             triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1760
1761             switch (scenario) {
1762                 case 'normal':
1763                     triggerString += "\\W+Build and Test.*"
1764                     break
1765
1766                 case 'corefx_innerloop': // maintain this asymmetry
1767                     triggerString += "\\W+CoreFX Tests.*"
1768                     break
1769
1770                 default:
1771                     triggerString += "\\W+${scenario}.*"
1772                     break
1773             }
1774
1775             triggerString += ".*"
1776             break
1777     }
1778
1779     // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1780     // non-flow jobs that are only used as part of flow jobs.
1781
1782     switch (architecture) {
1783         case 'x64': // editor brace matching: {
1784             if (scenario == 'formatting') {
1785                 assert configuration == 'Checked'
1786                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1787                     isDefaultTrigger = true
1788                     contextString = "${os} ${architecture} Formatting"
1789                 }
1790                 break
1791             }
1792
1793             switch (os) {
1794                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1795                 case 'Debian8.4':
1796                 case 'RHEL7.2':
1797                     if (scenario == 'innerloop') {
1798                         assert !isFlowJob
1799                         contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1800                         isDefaultTrigger = true
1801                         break
1802                     }
1803
1804                     // fall through
1805
1806                 case 'Fedora24':
1807                 case 'Ubuntu16.04':
1808                 case 'Ubuntu16.10':
1809                     assert !isFlowJob
1810                     assert scenario != 'innerloop'
1811                     contextString = "${os} ${architecture} ${configuration} Build"
1812                     triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1813                     break
1814
1815                 case 'Ubuntu':
1816                     if (scenario == 'illink') {
1817                         break
1818                     }
1819                     else if (scenario == 'corefx_innerloop') {
1820                         if (configuration == 'Checked') {
1821                             isDefaultTrigger = true
1822                         }
1823                         break
1824                     }
1825
1826                     // fall through
1827
1828                 case 'OSX10.12':
1829                     // Triggers on the non-flow jobs aren't necessary here
1830                     // Corefx testing uses non-flow jobs.
1831                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1832                         needsTrigger = false
1833                         break
1834                     }
1835                     switch (scenario) {
1836                         case 'innerloop':
1837                             isDefaultTrigger = true
1838                             break
1839
1840                         case 'no_tiered_compilation_innerloop':
1841                             if (os == 'Ubuntu') {
1842                                 isDefaultTrigger = true
1843                             }
1844                             break
1845
1846                         default:
1847                             break
1848                     }
1849                     break
1850
1851                 case 'CentOS7.1':
1852                     switch (scenario) {
1853                         case 'innerloop':
1854                             // CentOS uses checked for default PR tests while debug is build only
1855                             if (configuration == 'Debug') {
1856                                 isDefaultTrigger = true
1857                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1858                                 break
1859                             }
1860
1861                             // Make sure this is a flow job to get build and test.
1862                             if (!isFlowJob) {
1863                                 needsTrigger = false
1864                                 break
1865                             }
1866
1867                             if (configuration == 'Checked') {
1868                                 assert job.name.contains("flow")
1869                                 isDefaultTrigger = true
1870                                 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1871                             }
1872                             break
1873
1874                         case 'normal':
1875                             // Make sure this is a flow job to get build and test.
1876                             if (!isFlowJob) {
1877                                 needsTrigger = false
1878                                 break
1879                             }
1880                             break
1881
1882                         default:
1883                             break
1884                     }
1885                     break
1886
1887                 case 'Windows_NT':
1888                     switch (scenario) {
1889                         case 'innerloop':
1890                         case 'no_tiered_compilation_innerloop':
1891                             isDefaultTrigger = true
1892                             break
1893
1894                         case 'corefx_innerloop':
1895                             if (configuration == 'Checked' || configuration == 'Release') {
1896                                 isDefaultTrigger = true
1897                             }
1898                             break
1899
1900                         default:
1901                             break
1902                     }
1903                     break
1904
1905                 default:
1906                     println("Unknown os: ${os}");
1907                     assert false
1908                     break
1909
1910             } // switch (os)
1911
1912             break
1913         // editor brace matching: }
1914
1915         case 'armel': // editor brace matching: {
1916             job.with {
1917                 publishers {
1918                     azureVMAgentPostBuildAction {
1919                         agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1920                     }
1921                 }
1922             }
1923
1924             switch (os) {
1925                 case 'Tizen':
1926                     if (scenario == 'innerloop') {
1927                         if (configuration == 'Checked') {
1928                             isDefaultTrigger = true
1929                         }
1930                     }
1931                     break
1932             }
1933
1934             break
1935         // editor brace matching: }
1936
1937         case 'arm':
1938         case 'arm64': // editor brace matching: {
1939
1940             switch (os) {
1941                 case 'Ubuntu':
1942                 case 'Ubuntu16.04':
1943
1944                     // Triggers on the non-flow jobs aren't necessary
1945                     if (!isFlowJob) {
1946                         needsTrigger = false
1947                         break
1948                     }
1949
1950                     switch (scenario) {
1951                         case 'innerloop':
1952                         case 'no_tiered_compilation_innerloop':
1953                             if (configuration == 'Checked') {
1954                                 isDefaultTrigger = true
1955                             }
1956                             break
1957                          case 'crossgen_comparison':
1958                             if (os == 'Ubuntu' && architecture == 'arm' && (configuration == 'Checked' || configuration == 'Release')) {
1959                                 isDefaultTrigger = true
1960                             }
1961                             break
1962                     }
1963                     break
1964
1965                 case 'Windows_NT':
1966                     assert isArmWindowsScenario(scenario)
1967
1968                     // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1969                     // the non-flow Build job. All others need a trigger on the flow job.
1970                     def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1971                     if (isFlowJob != needsFlowJobTrigger) {
1972                         needsTrigger = false
1973                         break
1974                     }
1975
1976                     switch (scenario) {
1977                         case 'innerloop':
1978                             if (configuration == 'Debug') {
1979                                 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
1980                                 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
1981                                 // People in arm64Users will get both this and the Checked Build and Test job.
1982                                 isDefaultTrigger = true
1983                             } else if (configuration == 'Checked') {
1984                                 isDefaultTrigger = true
1985                                 isArm64PrivateJob = true
1986                             }
1987                             break
1988                         default:
1989                             isArm64PrivateJob = true
1990                             break
1991                     }
1992                     break
1993                 default:
1994                     println("NYI os: ${os}");
1995                     assert false
1996                     break
1997             }
1998             break
1999
2000         // editor brace matching: }
2001         case 'x86': // editor brace matching: {
2002             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2003             if (os == 'Ubuntu') {
2004                 // Triggers on the non-flow jobs aren't necessary here
2005                 if (!isFlowJob) {
2006                     needsTrigger = false
2007                     break
2008                 }
2009
2010                 // on-demand only for ubuntu x86
2011                 contextString = "${os} ${architecture} ${configuration} Build"
2012                 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2013                 break
2014             }
2015             switch (scenario) {
2016                 case 'innerloop':
2017                 case 'no_tiered_compilation_innerloop':
2018                     isDefaultTrigger = true
2019                     break
2020                 default:
2021                     break
2022             }
2023             break
2024
2025         // editor brace matching: }
2026         case 'x64_arm64_altjit':
2027         case 'x86_arm_altjit':
2028             // Everything default
2029             break
2030
2031         default:
2032             println("Unknown architecture: ${architecture}");
2033             assert false
2034             break
2035     }
2036
2037     if (needsTrigger) {
2038         if (isArm64PrivateJob) {
2039             if (isDefaultTrigger) {
2040                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2041             }
2042             else {
2043                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2044             }
2045         }
2046         else {
2047             if (isDefaultTrigger) {
2048                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2049             }
2050             else {
2051                 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2052             }
2053         }
2054     }
2055 }
2056
2057 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2058     def buildCommands = []
2059     def osGroup = getOSGroup(os)
2060     def lowerConfiguration = configuration.toLowerCase()
2061
2062     // Which set of tests to build? Innerloop tests build Pri-0.
2063     // Currently, we only generate asm diffs on Pri-0 tests, if we generate asm diffs on tests at all.
2064     // CoreFX testing skipts building tests altogether (done below).
2065     // All other scenarios build Pri-1 tests.
2066     def priority = '1'
2067     if (isInnerloopTestScenario(scenario)) {
2068         priority = '0'
2069     }
2070
2071     def doCoreFxTesting = isCoreFxScenario(scenario)
2072
2073     def buildCoreclrTests = true
2074     if (doCoreFxTesting || (scenario == 'pmi_asm_diffs')) {
2075         // These scenarios don't need the coreclr tests build.
2076         buildCoreclrTests = false
2077     }
2078
2079     // Calculate the build steps, archival, and xunit results
2080     switch (os) {
2081         case 'Windows_NT': // editor brace matching: {
2082             switch (architecture) {
2083                 case 'x64':
2084                 case 'x86':
2085                 case 'x86_arm_altjit':
2086                 case 'x64_arm64_altjit':
2087                     def arch = architecture
2088                     def buildOpts = ''
2089                     if (architecture == 'x86_arm_altjit') {
2090                         arch = 'x86'
2091                     }
2092                     else if (architecture == 'x64_arm64_altjit') {
2093                         arch = 'x64'
2094                     }
2095
2096                     if (scenario == 'formatting') {
2097                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2098                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2099                         break
2100                     }
2101
2102                     if (scenario == 'illink') {
2103                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2104                     }
2105
2106                     // If it is a release build for Windows, ensure PGO is used, else fail the build.
2107                     if ((lowerConfiguration == 'release') &&
2108                         (scenario in Constants.basicScenarios) &&
2109                         (architecture != 'x86_arm_altjit') &&
2110                         (architecture != 'x64_arm64_altjit')) {
2111
2112                         buildOpts += ' -enforcepgo'
2113                     }
2114
2115                     if (buildCoreclrTests) {
2116                         buildOpts += " -priority=${priority}"
2117                     } else {
2118                         buildOpts += ' skiptests';
2119                     }
2120
2121                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2122                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2123                     // 35 characters long.
2124
2125                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2126
2127                     if (scenario == 'pmi_asm_diffs') {
2128                         // Now, generate the layout. We don't have any tests, so we need to do some annoying magic before calling runtest.cmd.
2129                         buildCommands += "run.cmd build -Project=\"tests\\build.proj\" -BuildOS=Windows_NT -BuildType=${lowerConfiguration} -BuildArch=${arch} -BatchRestorePackages"
2130                         buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${arch} GenerateLayoutOnly"
2131
2132                         // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2133                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-pmi-diffs.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration}"
2134
2135                         // ZIP up the asm
2136                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('_\\_asm', '.\\dasm.${os}.${architecture}.${configuration}.zip')\"";
2137
2138                         // Archive the asm
2139                         Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip")
2140                         break
2141                     }
2142
2143                     if (!isBuildOnly) {
2144                         def runtestArguments = ''
2145                         def testOpts = 'collectdumps'
2146
2147                         if (isR2RScenario(scenario)) {
2148
2149                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2150                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2151                             // to cause the tests to be crossgen'ed.
2152
2153                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2154                                 testOpts += ' crossgenaltjit protononjit.dll'
2155                             } else {
2156                                 testOpts += ' crossgen'
2157                             }
2158
2159                             testOpts += ' runcrossgentests'
2160                         }
2161                         else if (scenario == 'jitdiff') {
2162                             testOpts += ' jitdisasm crossgen'
2163                         }
2164                         else if (scenario == 'ilrt') {
2165                             testOpts += ' ilasmroundtrip'
2166                         }
2167                         else if (isLongGc(scenario)) {
2168                             testOpts += " ${scenario} sequential"
2169                         }
2170                         else if (scenario == 'standalone_gc') {
2171                             testOpts += ' gcname clrgc.dll'
2172                         }
2173                         else if (scenario == 'illink') {
2174                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2175                         }
2176
2177                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2178                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2179                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2180                         // which is read by the xunit harness.)
2181                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2182                         {
2183                             def timeout = 1800000
2184                             testOpts += " timeout ${timeout}"
2185                         }
2186
2187                         // If we are running a stress mode, we should write out the set of key
2188                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2189
2190                         def envScriptPath = ''
2191                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2192                             def buildCommandsStr = ''
2193                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2194                             buildCommandsStr += envScriptCreate(os, envScriptPath)
2195
2196                             if (isJitStressScenario(scenario)) {
2197                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2198                             }
2199                             else if (isR2RStressScenario(scenario)) {
2200                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2201                             }
2202
2203                             if (architecture == 'x86_arm_altjit') {
2204                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2205                             }
2206                             else if (architecture == 'x64_arm64_altjit') {
2207                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2208                             }
2209
2210                             envScriptFinalize(os, envScriptPath)
2211
2212                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2213                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2214                             // into a single script.
2215                             buildCommands += buildCommandsStr
2216                         }
2217                         else if (architecture == 'x86_arm_altjit') {
2218                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2219                         }
2220                         else if (architecture == 'x64_arm64_altjit') {
2221                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2222                         }
2223                         if (envScriptPath != '') {
2224                             testOpts += " TestEnv ${envScriptPath}"
2225                         }
2226
2227                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2228
2229                         if (doCoreFxTesting) {
2230                             if (scenario == 'corefx_innerloop') {
2231                                 // Create CORE_ROOT and testhost
2232                                 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2233                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2234
2235                                 // Archive and process (only) the test results
2236                                 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2237                                 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2238                             }
2239                             else {
2240                                 def workspaceRelativeFxRoot = "_/fx"
2241                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2242                                 def fxBranch = getFxBranch(branch)
2243
2244                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2245
2246                                 // Archive and process (only) the test results
2247                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2248                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2249
2250                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2251                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2252                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2253                             }
2254                         }
2255                         else if (isGcReliabilityFramework(scenario)) {
2256                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2257                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2258                         }
2259                         else {
2260                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2261                         }
2262                     } // end if (!isBuildOnly)
2263
2264                     if (!doCoreFxTesting) {
2265                         // Run the rest of the build
2266                         // Build the mscorlib for the other OS's
2267                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2268                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2269
2270                         if (arch == 'x64') {
2271                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2272                         }
2273
2274                         // Zip up the tests directory so that we don't use so much space/time copying
2275                         // 10s of thousands of files around.
2276                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2277
2278                         if (!isJitStressScenario(scenario)) {
2279                             // For Windows, pull full test results and test drops for x86/x64.
2280                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2281                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2282                         }
2283
2284                         if (scenario == 'jitdiff') {
2285                             // retrieve jit-dasm output for base commit, and run jit-diff
2286                             if (!isBuildOnly) {
2287                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2288                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2289                             }
2290                         }
2291
2292                         if (!isBuildOnly) {
2293                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2294                         }
2295                     }
2296                     break
2297                 case 'arm':
2298                 case 'arm64':
2299                     assert isArmWindowsScenario(scenario)
2300
2301                     def buildOpts = ''
2302
2303                     if (buildCoreclrTests) {
2304                         buildOpts += " -priority=${priority}"
2305                     } else {
2306                         buildOpts += ' skiptests'
2307                     }
2308
2309                     // This is now a build only job. Do not run tests. Use the flow job.
2310                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2311
2312                     if (doCoreFxTesting) {
2313                         assert isBuildOnly
2314
2315                         // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2316                         def envScriptPath = ''
2317                         def buildCommandsStr = ''
2318                         envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2319                         buildCommandsStr += envScriptCreate(os, envScriptPath)
2320                         buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2321                         envScriptFinalize(os, envScriptPath)
2322                         buildCommands += buildCommandsStr
2323
2324                         def workspaceRelativeFxRootLinux = "_/fx"
2325                         def workspaceRelativeFxRootWin = "_\\fx"
2326                         def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2327                         def fxBranch = getFxBranch(branch)
2328
2329                         buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2330
2331                         // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2332                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2333                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2334
2335                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2336                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2337                     } else {
2338                         // Zip up the tests directory so that we don't use so much space/time copying
2339                         // 10s of thousands of files around.
2340                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2341
2342                         // Add archival.
2343                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2344                     }
2345                     break
2346                 default:
2347                     println("Unknown architecture: ${architecture}");
2348                     assert false
2349                     break
2350             }
2351             break
2352         // end case 'Windows_NT'; editor brace matching: }
2353         case 'Ubuntu':
2354         case 'Ubuntu16.04':
2355         case 'Ubuntu16.10':
2356         case 'Debian8.4':
2357         case 'OSX10.12':
2358         case 'CentOS7.1':
2359         case 'RHEL7.2':
2360         case 'Tizen':
2361         case 'Fedora24': // editor brace matching: {
2362             switch (architecture) {
2363                 case 'x64':
2364                 case 'x86':
2365                     if (architecture == 'x86' && os == 'Ubuntu') {
2366                         // build and PAL test
2367                         def dockerImage = getDockerImageName(architecture, os, true)
2368                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2369                         dockerImage = getDockerImageName(architecture, os, false)
2370                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2371                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2372                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2373                         break
2374                     }
2375
2376                     if (scenario == 'formatting') {
2377                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2378                         Utilities.addArchival(newJob, "format.patch", "", true, false)
2379                         break
2380                     }
2381
2382                     if (scenario == 'pmi_asm_diffs') {
2383                         buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests skipbuildpackages"
2384                         buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatelayoutonly"
2385
2386                         // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2387                         buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration}"
2388
2389                         // ZIP up the asm
2390                         buildCommands += "zip -r dasm.${os}.${architecture}.${configuration}.zip ./_/_asm"
2391
2392                         // Archive the asm
2393                         Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip")
2394                         break
2395                     }
2396
2397                     if (scenario == 'illink') {
2398                         assert(os == 'Ubuntu')
2399                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2400                     }
2401
2402                     if (!doCoreFxTesting) {
2403                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2404                         // only on supported OS platforms.
2405                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2406                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2407
2408                         buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2409                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2410
2411                         // Basic archiving of the build
2412                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2413                         // And pal tests
2414                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2415                     }
2416                     else {
2417                         if (scenario == 'corefx_innerloop') {
2418                             assert os == 'Ubuntu' || 'OSX10.12'
2419                             assert architecture == 'x64'
2420
2421                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2422                             buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2423                             buildCommands += "./tests/runtest.sh ${lowerConfiguration} --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2424
2425                             break
2426                             // Archive and process (only) the test results
2427                             Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2428                             Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2429                         }
2430                         else {
2431                             // Corefx stress testing
2432                             assert os == 'Ubuntu'
2433                             assert architecture == 'x64'
2434                             assert lowerConfiguration == 'checked'
2435                             assert isJitStressScenario(scenario)
2436
2437                             // Build coreclr
2438                             buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2439
2440                             def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2441
2442                             def envScriptCmds = envScriptCreate(os, scriptFileName)
2443                             envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2444                             envScriptCmds += envScriptFinalize(os, scriptFileName)
2445                             buildCommands += envScriptCmds
2446
2447                             // Build and text corefx
2448                             def workspaceRelativeFxRoot = "_/fx"
2449                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2450                             def fxBranch = getFxBranch(branch)
2451
2452                             buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2453
2454                             // Archive and process (only) the test results
2455                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2456                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2457                         }
2458                     }
2459                     break
2460                 case 'armem':
2461                     // Emulator cross builds for ARM runs on Tizen currently
2462                     assert os == 'Tizen'
2463
2464                     def arm_abi = "armel"
2465                     def linuxCodeName = "tizen"
2466
2467                     // Unzip the Windows test binaries first. Exit with 0
2468                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2469
2470                     // Unpack the corefx binaries
2471                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2472                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2473
2474                     // Call the ARM CI script to cross build and test using docker
2475                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2476                     --mode=docker \\
2477                     --${arm_abi} \\
2478                     --linuxCodeName=${linuxCodeName} \\
2479                     --buildConfig=${lowerConfiguration} \\
2480                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2481                     --coreFxBinDir=./bin/CoreFxBinDir \\
2482                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2483
2484                     // Basic archiving of the build, no pal tests
2485                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2486                     break
2487                 case 'arm64':
2488                 case 'arm':
2489                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
2490                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2491
2492                     // Add some useful information to the log file. Ignore return codes.
2493                     buildCommands += "uname -a || true"
2494
2495                     def additionalOpts = ""
2496                     if (architecture == 'arm') {
2497                         additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2498                     }
2499
2500                     // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2501                     // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2502                     // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2503                     // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2504                     // ZIP up the generated CoreFX runtime and tests.
2505
2506                     def dockerImage = getDockerImageName(architecture, os, true)
2507                     def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2508
2509                     buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2510
2511                     if (doCoreFxTesting) {
2512                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2513
2514                         def envScriptCmds = envScriptCreate(os, scriptFileName)
2515                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2516                         envScriptCmds += envScriptFinalize(os, scriptFileName)
2517                         buildCommands += envScriptCmds
2518
2519                         // Build and text corefx
2520                         def workspaceRelativeFxRootLinux = "_/fx"
2521                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2522                         def fxBranch = getFxBranch(branch)
2523
2524                         buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2525
2526                         // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2527                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2528                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2529
2530                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2531                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2532                     }
2533                     else if (isCrossGenComparisonScenario(scenario)) {
2534                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2535
2536                         def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2537                         def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2538                         def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2539                         def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2540                         def workspaceRelativeResultsDir = "_"
2541                         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2542                         def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2543                         getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
2544                             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2545                             def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2546
2547                             buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2548                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2549                             buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2550                         } // crossArch
2551                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2552                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2553                     }
2554                     else if (scenario == 'pmi_asm_diffs') {
2555                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2556
2557                         // Pass `-skip_diffs` -- the actual diffs will be done on an arm machine in the test job. This is the build job.
2558                         // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2559                         buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -skip_diffs True"
2560
2561                         // ZIP what we created.
2562                         buildCommands += "zip -r coreroot.${os}.${architecture}.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2563                         buildCommands += "zip -r coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.zip ./_/_c/bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2564
2565                         // Archive the built artifacts
2566                         Utilities.addArchival(newJob, "coreroot.${os}.${architecture}.${lowerConfiguration}.zip,coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.zip")
2567                     }
2568                     else if (architecture == 'arm') {
2569                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2570                         // download the appropriate CoreFX packages.
2571                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2572                         // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2573
2574                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2575
2576                         // ZIP up for the test job (created in the flow job code):
2577                         // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2578                         //     used by runtest.sh as the "--coreOverlayDir" argument.
2579                         // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2580                         //     used by runtest.sh as the "--testNativeBinDir" argument.
2581
2582                         // These commands are assumed to be run from the root of the workspace.
2583                         buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2584                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2585
2586                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2587                     }
2588                     else {
2589                         assert architecture == 'arm64'
2590
2591                         // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2592                         // Linux/arm64 does not use Windows-built tests.
2593
2594                         def testBuildOpts = ""
2595                         if (priority == '1') {
2596                             testBuildOpts = "priority1"
2597                         }
2598
2599                         buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2600
2601                         // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2602                         buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
2603
2604                         // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
2605                         // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
2606                         buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2607
2608                         Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2609                     }
2610
2611                     // Archive the build logs from both product and test builds.
2612                     Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
2613
2614                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2615                     // the cleanup task in Jenkins can't remove.
2616                     newJob.with {
2617                         publishers {
2618                             azureVMAgentPostBuildAction {
2619                                 agentPostBuildAction('Delete agent after build execution (when idle).')
2620                             }
2621                         }
2622                     }
2623                     break
2624                 default:
2625                     println("Unknown architecture: ${architecture}");
2626                     assert false
2627                     break
2628             }
2629             break
2630         // editor brace matching: }
2631         default:
2632             println("Unknown os: ${os}");
2633             assert false
2634             break
2635     } // os
2636
2637     return buildCommands
2638 }
2639
2640 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2641 // Returns true if the job should be generated.
2642 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2643 {
2644     // The various "innerloop" jobs are only available as PR triggered.
2645
2646     if (!isPR) {
2647         if (isInnerloopTestScenario(scenario)) {
2648             return false
2649         }
2650
2651         if (scenario == 'corefx_innerloop') {
2652             return false
2653         }
2654     }
2655
2656     // Tizen is only supported for armem architecture
2657     if (os == 'Tizen' && architecture != 'armem') {
2658         return false
2659     }
2660
2661     // Filter based on architecture.
2662
2663     switch (architecture) {
2664         case 'arm':
2665             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2666                 return false
2667             }
2668             break
2669         case 'arm64':
2670             if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2671                 return false
2672             }
2673             break
2674         case 'armem':
2675             if (os != 'Tizen') {
2676                 return false
2677             }
2678             break
2679         case 'x86_arm_altjit':
2680         case 'x64_arm64_altjit':
2681             if (os != 'Windows_NT') {
2682                 return false
2683             }
2684             break
2685         case 'x86':
2686             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2687                 return false
2688             }
2689             break
2690         case 'x64':
2691             // Everything implemented
2692             break
2693         default:
2694             println("Unknown architecture: ${architecture}")
2695             assert false
2696             break
2697     }
2698
2699     // Which (Windows) build only jobs are required?
2700
2701     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2702
2703     if (isBuildOnly) {
2704         switch (architecture) {
2705             case 'arm':
2706             case 'arm64':
2707                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2708                 if (!isCoreFxScenario(scenario)) {
2709                     return false
2710                 }
2711                 break
2712             case 'x64':
2713             case 'x86':
2714                 if (!isNormalOrInnerloop) {
2715                     return false
2716                 }
2717                 break
2718             default:
2719                 return false
2720         }
2721     }
2722
2723     // Filter based on scenario.
2724
2725     if (isJitStressScenario(scenario)) {
2726         if (configuration != 'Checked') {
2727             return false
2728         }
2729
2730         def isEnabledOS = (os == 'Windows_NT') ||
2731                           (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2732                           (os == 'Ubuntu' && architecture == 'arm') ||
2733                           (os == 'Ubuntu16.04' && architecture == 'arm64')
2734         if (!isEnabledOS) {
2735             return false
2736         }
2737
2738         switch (architecture) {
2739             case 'x64':
2740             case 'x86_arm_altjit':
2741             case 'x64_arm64_altjit':
2742                 break
2743
2744             case 'x86':
2745                 // x86 ubuntu: no stress modes
2746                 if (os == 'Ubuntu') {
2747                     return false
2748                 }
2749                 break
2750
2751             case 'arm':
2752             case 'arm64':
2753                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2754                 // No "regular" Windows arm corefx jobs, e.g.
2755                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2756                 // the Ubuntu arm "regular" jobs don't run tests anyway).
2757                 if (os == 'Windows_NT') {
2758                     if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2759                         return false
2760                     }
2761                 }
2762                 else {
2763                     if (!isCoreFxScenario(scenario)) {
2764                         return false
2765                     }
2766                 }
2767                 break
2768
2769             default:
2770                 // armem: no stress jobs for ARM emulator.
2771                 return false
2772         }
2773     }
2774     else if (isR2RScenario(scenario)) {
2775         if (os != 'Windows_NT') {
2776             return false
2777         }
2778
2779         if (isR2RBaselineScenario(scenario)) {
2780             // no need for Debug scenario; Checked is sufficient
2781             if (configuration != 'Checked' && configuration != 'Release') {
2782                 return false
2783             }
2784         }
2785         else if (isR2RStressScenario(scenario)) {
2786             // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2787             if (configuration != 'Checked') {
2788                 return false
2789             }
2790         }
2791
2792         switch (architecture) {
2793             case 'arm':
2794             case 'arm64':
2795                 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2796                 return false
2797
2798             default:
2799                 break
2800         }
2801     }
2802     else if (isCrossGenComparisonScenario(scenario)) {
2803         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2804     }
2805     else {
2806         // Skip scenarios
2807         switch (scenario) {
2808             case 'ilrt':
2809                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2810                 // the default scenario build
2811                 if (os != 'Windows_NT') {
2812                     return false
2813                 }
2814                 // Only x64 for now
2815                 if (architecture != 'x64') {
2816                     return false
2817                 }
2818                 // Release only
2819                 if (configuration != 'Release') {
2820                     return false
2821                 }
2822                 break
2823             case 'jitdiff':
2824                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2825                     return false
2826                 }
2827                 if (architecture != 'x64') {
2828                     return false
2829                 }
2830                 if (configuration != 'Checked') {
2831                     return false
2832                 }
2833                 break
2834             case 'longgc':
2835             case 'gcsimulator':
2836                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2837                     return false
2838                 }
2839                 if (architecture != 'x64') {
2840                     return false
2841                 }
2842                 if (configuration != 'Release') {
2843                     return false
2844                 }
2845                 break
2846             case 'gc_reliability_framework':
2847             case 'standalone_gc':
2848                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2849                     return false
2850                 }
2851
2852                 if (architecture != 'x64') {
2853                     return false
2854                 }
2855
2856                 if (configuration != 'Release' && configuration != 'Checked') {
2857                     return false
2858                 }
2859                 break
2860             // We only run Windows and Ubuntu x64 Checked for formatting right now
2861             case 'formatting':
2862                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2863                     return false
2864                 }
2865                 if (architecture != 'x64') {
2866                     return false
2867                 }
2868                 if (configuration != 'Checked') {
2869                     return false
2870                 }
2871                 break
2872             case 'illink':
2873                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2874                     return false
2875                 }
2876                 if (architecture != 'x64' && architecture != 'x86') {
2877                     return false
2878                 }
2879                 break
2880             case 'normal':
2881                 // Nothing skipped
2882                 break
2883             case 'innerloop':
2884                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2885                     return false
2886                 }
2887                 break
2888             case 'corefx_innerloop':
2889                 if (os != 'Windows_NT' && os != 'Ubuntu' &&  os != 'OSX10.12') {
2890                     return false
2891                 }
2892                 if (architecture != 'x64') {
2893                     return false
2894                 }
2895                 break
2896             case 'pmi_asm_diffs':
2897                 if (configuration != 'Checked') {
2898                     return false
2899                 }
2900                 if (architecture == 'armem') {
2901                     return false
2902                 }
2903                 // Currently, we don't support pmi_asm_diffs for Windows arm/arm64. Is is not in validArmWindowsScenarios.
2904                 if ((os == 'Windows_NT') && (architecture == 'arm' || architecture == 'arm64')) {
2905                     return false
2906                 }
2907                 // Currently, no support for Linux x86.
2908                 if ((os != 'Windows_NT') && (architecture == 'x86')) {
2909                     return false
2910                 }
2911                 break
2912             default:
2913                 println("Unknown scenario: ${scenario}")
2914                 assert false
2915                 break
2916         }
2917     }
2918
2919     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2920     // runtime behavior, not compile-time behavior, are not interesting.
2921     switch (architecture) {
2922         case 'x86_arm_altjit':
2923         case 'x64_arm64_altjit':
2924             if (isGCStressRelatedTesting(scenario)) {
2925                 return false
2926             }
2927             break
2928         default:
2929             break
2930     }
2931
2932     // The job was not filtered out, so we should generate it!
2933     return true
2934 }
2935
2936 Constants.allScenarios.each { scenario ->
2937     [true, false].each { isPR ->
2938         Constants.architectureList.each { architecture ->
2939             Constants.configurationList.each { configuration ->
2940                 Constants.osList.each { os ->
2941                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2942                     // and reset the os to Windows_NT
2943                     def isBuildOnly = false
2944                     if (os == 'Windows_NT_BuildOnly') {
2945                         isBuildOnly = true
2946                         os = 'Windows_NT'
2947                     }
2948
2949                     if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2950                         return
2951                     }
2952
2953                     // Calculate names
2954                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2955                     def folderName = getJobFolder(scenario)
2956
2957                     // Create the new job
2958                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2959
2960                     addToViews(newJob, false, isPR, architecture, os, configuration, scenario) // isFlowJob == false
2961
2962                     setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2963
2964                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2965                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2966                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2967
2968                     // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2969                     // We don't use a flow job for this, but we do depend on there being existing builds with these
2970                     // artifacts produced.
2971                     if ((architecture == 'armem') && (os == 'Tizen')) {
2972                         // Define the Windows Tests and Corefx build job names
2973                         def lowerConfiguration = configuration.toLowerCase()
2974                         def WindowsTestsName = projectFolder + '/' +
2975                                                Utilities.getFullJobName(project,
2976                                                                         getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2977                                                                         false)
2978                         def fxBranch = getFxBranch(branch)
2979                         def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2980                                            Utilities.getFolderName(fxBranch)
2981
2982                         def arm_abi = 'armel'
2983                         def corefx_os = 'tizen'
2984
2985                         // Let's use release CoreFX to test checked CoreCLR,
2986                         // because we do not generate checked CoreFX in CoreFX CI yet.
2987                         def corefx_lowerConfiguration = lowerConfiguration
2988                         if (lowerConfiguration == 'checked') {
2989                             corefx_lowerConfiguration = 'release'
2990                         }
2991
2992                         // Copy the Windows test binaries and the Corefx build binaries
2993                         newJob.with {
2994                             steps {
2995                                 copyArtifacts(WindowsTestsName) {
2996                                     includePatterns('bin/tests/tests.zip')
2997                                     buildSelector {
2998                                         latestSuccessful(true)
2999                                     }
3000                                 }
3001                                 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
3002                                     includePatterns('bin/build.tar.gz')
3003                                     buildSelector {
3004                                         latestSuccessful(true)
3005                                     }
3006                                 }
3007                             } // steps
3008                         } // newJob.with
3009                     }
3010
3011                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
3012
3013                     newJob.with {
3014                         steps {
3015                             if (os == 'Windows_NT') {
3016                                 buildCommands.each { buildCommand ->
3017                                     batchFile(buildCommand)
3018                                 }
3019                             }
3020                             else {
3021                                 buildCommands.each { buildCommand ->
3022                                     shell(buildCommand)
3023                                 }
3024                             }
3025                         } // steps
3026                     } // newJob.with
3027
3028                 } // os
3029             } // configuration
3030         } // architecture
3031     } // isPR
3032 } // scenario
3033
3034 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
3035 // Returns the newly created job.
3036 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3037 {
3038     def osGroup = getOSGroup(os)
3039     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3040
3041     def jobFolder = getJobFolder(scenario)
3042     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3043         parameters {
3044             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3045         }
3046
3047         steps {
3048             // Set up the copies
3049
3050             // Coreclr build we are trying to test
3051             //
3052             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3053
3054             copyArtifacts(inputCoreCLRBuildName) {
3055                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3056                 buildSelector {
3057                     buildNumber('${CORECLR_BUILD}')
3058                 }
3059             }
3060
3061             if (isCoreFxScenario(scenario)) {
3062
3063                 // Only arm/arm64 supported for corefx testing now.
3064                 assert architecture == 'arm' || architecture == 'arm64'
3065
3066                 // Unzip CoreFx runtime
3067                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3068
3069                 // Unzip CoreFx tests.
3070                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3071
3072                 // Add the script to run the corefx tests
3073                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3074                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\bin\\tests"
3075                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3076                 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3077
3078             } else { // !isCoreFxScenario(scenario)
3079
3080                 // Unzip tests.
3081                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3082
3083                 def buildCommands = ""
3084
3085                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3086                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3087                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3088
3089                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3090                 addCommand("SETLOCAL ENABLEEXTENSIONS")
3091
3092                 // For all jobs
3093                 addEnvVariable("CORE_ROOT", coreRootLocation)
3094                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3095                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3096
3097                 // If we are running a stress mode, we'll set those variables as well
3098                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3099                     def stressValues = null
3100                     if (isJitStressScenario(scenario)) {
3101                         stressValues = Constants.jitStressModeScenarios[scenario]
3102                     }
3103                     else {
3104                         stressValues = Constants.r2rStressScenarios[scenario]
3105                     }
3106
3107                     stressValues.each { key, value ->
3108                         addEnvVariable(key, value)
3109                     }
3110                 }
3111
3112                 if (isR2RScenario(scenario)) {
3113                     // Crossgen the framework assemblies.
3114                     buildCommands += """
3115 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3116 @goto skip_PrecompileAssembly
3117
3118 :PrecompileAssembly
3119 @REM Skip mscorlib since it is already precompiled.
3120 @if /I "%3" == "mscorlib.dll" exit /b 0
3121 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3122
3123 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3124 @if "%errorlevel%" == "-2146230517" (
3125     echo %2 is not a managed assembly.
3126 ) else if "%errorlevel%" == "-2146234344" (
3127     echo %2 is not a managed assembly.
3128 ) else if %errorlevel% neq 0 (
3129     echo Unable to precompile %2
3130 ) else (
3131     echo Precompiled %2
3132 )
3133 @exit /b 0
3134
3135 :skip_PrecompileAssembly
3136 """
3137
3138                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3139                     // crossgen on tests before running them.
3140                     addEnvVariable("RunCrossGen", "true")
3141                 } // isR2RScenario(scenario)
3142
3143                 // Run runtest.cmd
3144                 // Do not run generate layout. It will delete the correct CORE_ROOT, and we do not have a correct product
3145                 // dir to copy from.
3146                 def runtestCommand = "%WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout"
3147
3148                 addCommand("${runtestCommand}")
3149
3150                 // Use the smarty errorlevel as the script errorlevel.
3151                 addCommand("exit /b %errorlevel%")
3152
3153                 batchFile(buildCommands)
3154             } // non-corefx testing
3155         } // steps
3156     } // job
3157
3158     if (!isCoreFxScenario(scenario)) {
3159         Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
3160     }
3161
3162     return newJob
3163 }
3164
3165 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3166 // E.g., non-Windows tests.
3167 // Returns the newly created job.
3168 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3169 {
3170     def lowerConfiguration = configuration.toLowerCase()
3171
3172     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3173     def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3174     def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3175
3176     def doCoreFxTesting = isCoreFxScenario(scenario)
3177     def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs')
3178
3179     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3180
3181     def osGroup = getOSGroup(os)
3182     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3183
3184     def testOpts = ''
3185     def useServerGC = false
3186
3187     // Enable Server GC for Ubuntu PR builds
3188     // REVIEW: why? Does this apply to all architectures? Why only PR?
3189     if (os == 'Ubuntu' && isPR) {
3190         testOpts += ' --useServerGC'
3191         useServerGC = true
3192     }
3193
3194     if (isR2RScenario(scenario)) {
3195
3196         testOpts += ' --crossgen --runcrossgentests'
3197
3198         if (scenario == 'r2r_jitstress1') {
3199             testOpts += ' --jitstress=1'
3200         }
3201         else if (scenario == 'r2r_jitstress2') {
3202             testOpts += ' --jitstress=2'
3203         }
3204         else if (scenario == 'r2r_jitstress1_tiered') {
3205             testOpts += ' --jitstress=1'
3206         }
3207         else if (scenario == 'r2r_jitstress2_tiered') {
3208             testOpts += ' --jitstress=2'
3209         }
3210         else if (scenario == 'r2r_jitstressregs1') {
3211             testOpts += ' --jitstressregs=1'
3212         }
3213         else if (scenario == 'r2r_jitstressregs2') {
3214             testOpts += ' --jitstressregs=2'
3215         }
3216         else if (scenario == 'r2r_jitstressregs3') {
3217             testOpts += ' --jitstressregs=3'
3218         }
3219         else if (scenario == 'r2r_jitstressregs4') {
3220             testOpts += ' --jitstressregs=4'
3221         }
3222         else if (scenario == 'r2r_jitstressregs8') {
3223             testOpts += ' --jitstressregs=8'
3224         }
3225         else if (scenario == 'r2r_jitstressregs0x10') {
3226             testOpts += ' --jitstressregs=0x10'
3227         }
3228         else if (scenario == 'r2r_jitstressregs0x80') {
3229             testOpts += ' --jitstressregs=0x80'
3230         }
3231         else if (scenario == 'r2r_jitstressregs0x1000') {
3232             testOpts += ' --jitstressregs=0x1000'
3233         }
3234         else if (scenario == 'r2r_jitminopts') {
3235             testOpts += ' --jitminopts'
3236         }
3237         else if (scenario == 'r2r_jitforcerelocs') {
3238             testOpts += ' --jitforcerelocs'
3239         }
3240         else if (scenario == 'r2r_gcstress15') {
3241             testOpts += ' --gcstresslevel=0xF'
3242         }
3243     }
3244     else if (scenario == 'jitdiff') {
3245         testOpts += ' --jitdisasm --crossgen'
3246     }
3247     else if (scenario == 'illink') {
3248         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3249     }
3250     else if (isLongGc(scenario)) {
3251         // Long GC tests behave very poorly when they are not
3252         // the only test running (many of them allocate until OOM).
3253         testOpts += ' --sequential'
3254
3255         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3256         // for running long GC and GCSimulator tests, respectively. We don't use them
3257         // here because using a playlist file produces much more readable output on the CI machines
3258         // and reduces running time.
3259         //
3260         // The Long GC playlist contains all of the tests that are
3261         // going to be run. The GCSimulator playlist contains all of
3262         // the GC simulator tests.
3263         if (scenario == 'longgc') {
3264             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3265         }
3266         else if (scenario == 'gcsimulator') {
3267             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3268         }
3269     }
3270     else if (isGcReliabilityFramework(scenario)) {
3271         testOpts += ' --build-overlay-only'
3272     }
3273     else if (scenario == 'standalone_gc') {
3274         if (osGroup == 'OSX') {
3275             testOpts += ' --gcname=libclrgc.dylib'
3276         }
3277         else if (osGroup == 'Linux') {
3278             testOpts += ' --gcname=libclrgc.so'
3279         }
3280         else {
3281             println("Unexpected OS group: ${osGroup} for os ${os}")
3282             assert false
3283         }
3284     }
3285
3286     // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3287     // will be null in this case.
3288
3289     def jobFolder = getJobFolder(scenario)
3290     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3291         parameters {
3292             if (inputTestsBuildName != null) {
3293                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3294             }
3295             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3296         }
3297
3298         steps {
3299             // Set up the copies
3300
3301             // Coreclr build containing the tests and mscorlib
3302             // pri1 jobs still need to copy windows_nt built tests
3303             if (inputTestsBuildName != null) {
3304                 copyArtifacts(inputTestsBuildName) {
3305                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3306                     buildSelector {
3307                         buildNumber('${CORECLR_WINDOWS_BUILD}')
3308                     }
3309                 }
3310             }
3311
3312             // Coreclr build we are trying to test
3313             //
3314             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3315             //
3316             // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3317             // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3318
3319             if (!isUbuntuArm64Job) {
3320                 copyArtifacts(inputCoreCLRBuildName) {
3321                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
3322                     buildSelector {
3323                         buildNumber('${CORECLR_BUILD}')
3324                     }
3325                 }
3326             }
3327
3328             if (isUbuntuArmJob) {
3329                 // Add some useful information to the log file. Ignore return codes.
3330                 shell("uname -a || true")
3331             }
3332
3333             if (isUbuntuArm64Job) {
3334                 // Copy the required artifacts directly, using wget, e.g.:
3335                 //
3336                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3337                 //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3338                 //
3339                 // parameterized as:
3340                 //
3341                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3342                 //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3343                 //
3344                 // CoreFX example artifact URLs:
3345                 //
3346                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3347                 //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3348                 //
3349                 // Note that the source might be in a "jitstress" folder.
3350                 //
3351                 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3352                 //
3353                 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3354
3355                 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3356
3357                 def mungedProjectName = Utilities.getFolderName(project)
3358                 def mungedBranchName = Utilities.getFolderName(branch)
3359
3360                 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3361                 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3362                 if (isPmiAsmDiffsScenario || doCoreFxTesting || doCrossGenComparison) {
3363                     // These depend on unique builds for each scenario
3364                     inputCoreCLRBuildScenario = scenario
3365                 }
3366                 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3367                 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3368
3369                 // Need to add the sub-folder if necessary.
3370                 def inputJobPath = "job/${inputJobName}"
3371                 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3372                 if (folderName != '') {
3373                     inputJobPath = "job/${folderName}/job/${inputJobName}"
3374                 }
3375
3376                 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3377
3378                 if (isPmiAsmDiffsScenario) {
3379                     def workspaceRelativeRootLinux = "_"
3380                     shell("mkdir -p ${workspaceRelativeRootLinux}")
3381                     shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.${os}.${architecture}.${lowerConfiguration}.zip")
3382                     shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.zip")
3383                 }
3384                 else if (doCoreFxTesting) {
3385                     shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3386                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3387                     shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3388                 }
3389                 else {
3390                     shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
3391                     shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
3392                 }
3393             }
3394
3395             if (architecture == 'x86') {
3396                 shell("mkdir ./bin/CoreFxNative")
3397
3398                 def fxBranch = getFxBranch(branch)
3399                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3400
3401                 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3402                     includePatterns('bin/build.tar.gz')
3403                     targetDirectory('bin/CoreFxNative')
3404                     buildSelector {
3405                         latestSuccessful(true)
3406                     }
3407                 }
3408
3409                 shell("mkdir ./bin/CoreFxBinDir")
3410                 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3411             }
3412
3413             if (isPmiAsmDiffsScenario) {
3414                 // TODO: add back "-q" when we know it works
3415                 shell("unzip -o ./coreroot.${os}.${architecture}.${lowerConfiguration}.zip || exit 0")
3416                 shell("unzip -o ./coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.zip || exit 0")
3417             }
3418             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3419             else if (doCoreFxTesting) {
3420                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3421                 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3422             }
3423             else if (architecture != 'arm64') {
3424                 // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
3425                 //
3426                 // Unzip the tests first.  Exit with 0
3427                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3428                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3429             }
3430
3431             // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3432             // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3433             // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3434             // However, it's believed that perhaps there's an issue with executable permission bits not getting
3435             // copied correctly.
3436             if (!doCoreFxTesting) {
3437                 if (isUbuntuArmJob) {
3438                     if (!isPmiAsmDiffsScenario) {
3439                         if (architecture == 'arm') {
3440                             shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3441                             shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3442                         }
3443                         else {
3444                             assert architecture == 'arm64'
3445                             shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
3446
3447                             // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
3448                             shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3449                         }
3450                     }
3451                 }
3452                 else {
3453                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3454                 }
3455             }
3456
3457             // Execute the tests
3458             def runDocker = isNeedDocker(architecture, os, false)
3459             def dockerPrefix = ""
3460             def dockerCmd = ""
3461             if (runDocker) {
3462                 def dockerImage = getDockerImageName(architecture, os, false)
3463                 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3464                 dockerCmd = dockerPrefix + "${dockerImage} "
3465             }
3466
3467             // If we are running a stress mode, we'll set those variables first.
3468             // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3469             if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3470                 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3471                 def envScriptCmds = envScriptCreate(os, scriptFileName)
3472                 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3473                 envScriptCmds += envScriptFinalize(os, scriptFileName)
3474                 shell("${envScriptCmds}")
3475                 testOpts += " --test-env=${scriptFileName}"
3476             }
3477
3478             // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3479             // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3480             // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3481             // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3482             if (isGCStressRelatedTesting(scenario)) {
3483                 if (architecture == 'x64') {
3484                     shell('./init-tools.sh')
3485                 }
3486             }
3487
3488             if (isPmiAsmDiffsScenario) {
3489                 shell("""\
3490 python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -skip_baseline_build True""")
3491
3492                 shell("zip -r dasm.${os}.${architecture}.${configuration}.zip ./_/_asm")
3493             }
3494             else if (doCoreFxTesting) {
3495                 shell("""\
3496 \${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3497             }
3498             else {
3499                 def runScript = "${dockerCmd}./tests/runtest.sh"
3500
3501                 // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
3502
3503                 shell("""\
3504 ${runScript} \\
3505     ${lowerConfiguration} \\
3506     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3507     --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3508     --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3509     --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3510             }
3511
3512             if (isGcReliabilityFramework(scenario)) {
3513                 // runtest.sh doesn't actually execute the reliability framework - do it here.
3514                 if (useServerGC) {
3515                     if (runDocker) {
3516                         dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3517                     }
3518                     else {
3519                         shell("export COMPlus_gcServer=1")
3520                     }
3521                 }
3522
3523                 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3524             }
3525         } // steps
3526     } // job
3527
3528     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3529     if (os in ['Ubuntu']) {
3530         SummaryBuilder summaries = new SummaryBuilder()
3531         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3532         summaries.emit(newJob)
3533     }
3534
3535     if (isPmiAsmDiffsScenario) {
3536         // Archive the asm
3537         Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip")
3538     }
3539     else if (doCoreFxTesting) {
3540         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3541         if ((os == "Ubuntu") && (architecture == 'arm')) {
3542             // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3543             //
3544             //   [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3545             //
3546             // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3547             // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3548         }
3549         else {
3550             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3551         }
3552     }
3553     else {
3554         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3555     }
3556
3557     return newJob
3558 }
3559
3560 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3561 {
3562     assert isCrossGenComparisonScenario(scenario)
3563
3564     def osGroup = getOSGroup(os)
3565     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3566
3567     def workspaceRelativeResultsDir = "_"
3568     def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3569
3570     def jobFolder = getJobFolder(scenario)
3571     def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3572         parameters {
3573             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3574         }
3575
3576         def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3577
3578         steps {
3579             copyArtifacts(inputCoreCLRBuildName) {
3580                 includePatterns("${workspaceRelativeArtifactsArchive}")
3581                 buildSelector {
3582                     buildNumber('${CORECLR_BUILD}')
3583                 }
3584             }
3585
3586             shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3587
3588             def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3589             def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3590             def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3591             def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3592
3593             def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3594             def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3595
3596             shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3597             shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3598             shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3599
3600             getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3601                 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3602                 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3603             } // crossArch
3604         } // steps
3605     }  // job
3606
3607     Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**")
3608     getCrossArchitectures(os, architecture, scenario).each{ crossArch ->
3609         def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3610         Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**")
3611     } // crossArch
3612
3613     return newJob
3614 }
3615
3616 // Create a test job that will be used by a flow job.
3617 // Returns the newly created job.
3618 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3619 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3620 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3621 {
3622     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3623
3624     def newJob = null
3625     if (windowsArmJob) {
3626         assert inputTestsBuildName == null
3627         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3628     }
3629     else if (isCrossGenComparisonScenario(scenario)) {
3630         assert inputTestsBuildName == null
3631         newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3632     }
3633     else {
3634         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3635     }
3636
3637     setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3638
3639     if (scenario == 'jitdiff') {
3640         def osGroup = getOSGroup(os)
3641         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3642     }
3643
3644     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3645     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3646
3647     return newJob
3648 }
3649
3650 // Create a flow job to tie together a build job with the given test job.
3651 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3652 // Returns the new flow job.
3653 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3654 {
3655     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3656     // Linux CoreCLR test
3657     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3658     def jobFolder = getJobFolder(scenario)
3659
3660     def newFlowJob = null
3661
3662     if (inputTestsBuildName == null) {
3663         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3664                         buildFlow("""\
3665 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3666
3667 // And then build the test build
3668 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3669 """)
3670         }
3671         JobReport.Report.addReference(inputCoreCLRBuildName)
3672         JobReport.Report.addReference(fullTestJobName)
3673     }
3674     else {
3675         newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3676                         buildFlow("""\
3677 // Build the input jobs in parallel
3678 parallel (
3679 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3680 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3681 )
3682
3683 // And then build the test build
3684 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3685                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3686 """)
3687         }
3688         JobReport.Report.addReference(inputCoreCLRBuildName)
3689         JobReport.Report.addReference(inputTestsBuildName)
3690         JobReport.Report.addReference(fullTestJobName)
3691     }
3692
3693     addToViews(newFlowJob, true, isPR, architecture, os, configuration, scenario) // isFlowJob = true
3694
3695     setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3696
3697     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3698     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3699
3700     return newFlowJob
3701 }
3702
3703 // Determine if we should generate a flow job for the given parameters.
3704 // Returns true if the job should be generated.
3705 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3706 {
3707     // The various "innerloop" jobs are only available as PR triggered.
3708
3709     if (!isPR) {
3710         if (isInnerloopTestScenario(scenario)) {
3711             return false
3712         }
3713
3714         if (scenario == 'corefx_innerloop') {
3715             return false
3716         }
3717     }
3718
3719     // Filter based on OS and architecture.
3720
3721     switch (architecture) {
3722         case 'arm':
3723             if (os != "Ubuntu" && os != "Windows_NT") {
3724                 return false
3725             }
3726             break
3727         case 'arm64':
3728             if (os != "Ubuntu16.04" && os != "Windows_NT") {
3729                 return false
3730             }
3731             break
3732         case 'x86':
3733             if (os != "Ubuntu") {
3734                 return false
3735             }
3736             break
3737         case 'x64':
3738             if (!(os in Constants.crossList)) {
3739                 return false
3740             }
3741             if (os == "Windows_NT") {
3742                 return false
3743             }
3744             break
3745         case 'armem':
3746         case 'x86_arm_altjit':
3747         case 'x64_arm64_altjit':
3748             // No flow jobs
3749             return false
3750         default:
3751             println("Unknown architecture: ${architecture}")
3752             assert false
3753             break
3754     }
3755
3756     def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3757
3758     // Filter based on scenario in OS.
3759
3760     if (os == 'Windows_NT') {
3761         assert architecture == 'arm' || architecture == 'arm64'
3762         if (!isArmWindowsScenario(scenario)) {
3763             return false
3764         }
3765         if (isNormalOrInnerloop && (configuration == 'Debug')) {
3766             // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3767             // To do that, it doesn't require a flow job.
3768             return false
3769         }
3770     }
3771     else {
3772         // Non-Windows
3773         if (architecture == 'arm') {
3774             if (!(scenario in Constants.validLinuxArmScenarios)) {
3775                 return false
3776             }
3777         }
3778         else if (architecture == 'arm64') {
3779             if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3780                 return false
3781             }
3782         }
3783         else if (architecture == 'x86') {
3784             // Linux/x86 only want innerloop and default test
3785             if (!isNormalOrInnerloop) {
3786                 return false
3787             }
3788         }
3789         else if (architecture == 'x64') {
3790             // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3791             // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3792             // build from a Windows machine.
3793             if (isCoreFxScenario(scenario)) {
3794                 return false
3795             }
3796         }
3797     }
3798
3799     // For CentOS, we only want Checked/Release builds.
3800     if (os == 'CentOS7.1') {
3801         if (configuration != 'Checked' && configuration != 'Release') {
3802             return false
3803         }
3804         if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3805             return false
3806         }
3807     }
3808
3809     // For RedHat and Debian, we only do Release builds.
3810     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3811         if (configuration != 'Release') {
3812             return false
3813         }
3814         if (!isNormalOrInnerloop) {
3815             return false
3816         }
3817     }
3818
3819     // Next, filter based on scenario.
3820
3821     if (isJitStressScenario(scenario)) {
3822         if (configuration != 'Checked') {
3823             return false
3824         }
3825     }
3826     else if (isR2RBaselineScenario(scenario)) {
3827         if (configuration != 'Checked' && configuration != 'Release') {
3828             return false
3829         }
3830     }
3831     else if (isR2RStressScenario(scenario)) {
3832         if (configuration != 'Checked') {
3833             return false
3834         }
3835     }
3836     else if (isCrossGenComparisonScenario(scenario)) {
3837         return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3838     }
3839     else {
3840         // Skip scenarios
3841         switch (scenario) {
3842             case 'ilrt':
3843             case 'longgc':
3844             case 'gcsimulator':
3845                 // Long GC tests take a long time on non-Release builds
3846                 // ilrt is also Release only
3847                 if (configuration != 'Release') {
3848                     return false
3849                 }
3850                 break
3851
3852             case 'jitdiff':
3853                 if (configuration != 'Checked') {
3854                     return false
3855                 }
3856                 break
3857
3858             case 'gc_reliability_framework':
3859             case 'standalone_gc':
3860                 if (configuration != 'Release' && configuration != 'Checked') {
3861                     return false
3862                 }
3863                 break
3864
3865             case 'formatting':
3866                 return false
3867
3868             case 'illink':
3869                 if (os != 'Windows_NT' && os != 'Ubuntu') {
3870                     return false
3871                 }
3872                 break
3873
3874             case 'normal':
3875                 // Nothing skipped
3876                 break
3877
3878             case 'innerloop':
3879                 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3880                     return false
3881                 }
3882                 break
3883
3884             case 'pmi_asm_diffs':
3885                 if (configuration != 'Checked') {
3886                     return false
3887                 }
3888                 // No need for flow job except for Linux arm/arm64
3889                 if ((os != 'Windows_NT') && (architecture != 'arm') && (architecture != 'arm64')) {
3890                     return false
3891                 }
3892                 break
3893
3894             case 'corefx_innerloop':
3895                 // No flow job needed
3896                 return false
3897
3898             default:
3899                 println("Unknown scenario: ${scenario}")
3900                 assert false
3901                 break
3902         }
3903     }
3904
3905     // The job was not filtered out, so we should generate it!
3906     return true
3907 }
3908
3909 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3910 Constants.allScenarios.each { scenario ->
3911     [true, false].each { isPR ->
3912         Constants.architectureList.each { architecture ->
3913             Constants.configurationList.each { configuration ->
3914                 Constants.osList.each { os ->
3915
3916                     if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3917                         return
3918                     }
3919
3920                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3921                     def doCoreFxTesting = isCoreFxScenario(scenario)
3922                     def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3923                     def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs')
3924
3925                     // Figure out the job name of the CoreCLR build the test will depend on.
3926
3927                     def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3928                     def inputCoreCLRBuildIsBuildOnly = false
3929                     if (doCoreFxTesting || isPmiAsmDiffsScenario) {
3930                         // Every CoreFx test depends on its own unique build.
3931                         inputCoreCLRBuildScenario = scenario
3932                         if (windowsArmJob) {
3933                             // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3934                             inputCoreCLRBuildIsBuildOnly = true
3935                         }
3936                     }
3937                     else if (doCrossGenComparison) {
3938                         inputCoreCLRBuildScenario = scenario
3939                     }
3940
3941                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3942                     def inputCoreCLRBuildName = projectFolder + '/' +
3943                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3944
3945                     // Figure out the name of the build job that the test job will depend on.
3946                     // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3947                     // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3948                     // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3949                     // CoreFX build creates the tests.
3950
3951                     def inputTestsBuildName = null
3952
3953                     // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
3954                     def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3955
3956                     if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job && !isPmiAsmDiffsScenario) {
3957                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3958
3959                         def inputTestsBuildArch = architecture
3960                         if (architecture == "arm") {
3961                             // Use the x86 test build for arm unix
3962                             inputTestsBuildArch = "x86"
3963                         }
3964
3965                         def inputTestsBuildIsBuildOnly = true
3966
3967                         inputTestsBuildName = projectFolder + '/' +
3968                             Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3969                     }
3970
3971                     // =============================================================================================
3972                     // Create the test job
3973                     // =============================================================================================
3974
3975                     def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3976
3977                     // =============================================================================================
3978                     // Create a build flow to join together the build and tests required to run this test.
3979                     // =============================================================================================
3980
3981                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
3982                         // Do not create the flow job for RHEL jobs.
3983                         return
3984                     }
3985
3986                     def fullTestJobName = projectFolder + '/' + testJob.name
3987                     def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3988
3989                 } // os
3990             } // configuration
3991         } // architecture
3992     } // isPR
3993 } // scenario
3994
3995 JobReport.Report.generateJobReport(out)
3996
3997 // Make the call to generate the help job
3998 Utilities.createHelperJob(this, project, branch,
3999     "Welcome to the ${project} Repository",  // This is prepended to the help message
4000     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
4001
4002 Utilities.addCROSSCheck(this, project, branch)