netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'default',
 139                'ilrt',
 140                'r2r',
 141                'longgc',
 142                'formatting',
 143                'gcsimulator',
 144                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 145                'standalone_gc',
 146                'gc_reliability_framework',
 147                'illink']
 148
 149     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 150
 151     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 152     // to Tests.lst file categories to exclude.
 153     //
 154     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 155     // and with the same values, with some commented out, for easier maintenance.
 156     //
 157     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 158     //
 159     def static validArmWindowsScenarios = [
 160                'default':                                [],
 161                // 'ilrt'
 162                'r2r':                                    ["R2R_FAIL"],
 163                // 'longgc'
 164                // 'formatting'
 165                // 'gcsimulator'
 166                // 'jitdiff'
 167                // 'standalone_gc'
 168                // 'gc_reliability_framework'
 169                // 'illink'
 170                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 184                'tieredcompilation':                      [],
 185                'forcerelocs':                            [],
 186                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 200                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 201                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 202                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 203                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 204                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 205                // 'jitsse2only'                          // Only relevant to xarch
 206                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 207                // 'corefx_baseline'
 208                // 'corefx_minopts'
 209                // 'corefx_tieredcompilation'
 210                // 'corefx_jitstress1'
 211                // 'corefx_jitstress2'
 212                // 'corefx_jitstressregs1'
 213                // 'corefx_jitstressregs2'
 214                // 'corefx_jitstressregs3'
 215                // 'corefx_jitstressregs4'
 216                // 'corefx_jitstressregs8'
 217                // 'corefx_jitstressregs0x10'
 218                // 'corefx_jitstressregs0x80'
 219                // 'corefx_jitstressregs0x1000'
 220                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 221                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 222                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 223                'heapverify1':                            [],
 224                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 225                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 226                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 227                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 230
 231                //
 232                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 233                //
 234
 235                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 236                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 254     ]
 255
 256     def static configurationList = ['Debug', 'Checked', 'Release']
 257
 258     // This is the set of architectures
 259     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 260 }
 261
 262 // **************************************************************
 263 // Create some specific views
 264 //
 265 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 266 // views based on a single regular expression. These views will be generated by adding a
 267 // specific set of jobs to them.
 268 //
 269 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 270 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 271 // those are required, don't add them (which simplifies the view pages, as well).
 272 // **************************************************************
 273
 274 class Views {
 275     def static MergeJobView = null
 276     def static PeriodicJobView = null
 277     def static ArchitectureViews = [:]
 278     def static OSViews = [:]
 279 }
 280
 281 // MergeJobView: include all jobs that execute when a PR change is merged.
 282 Views.MergeJobView = listView('Merge') {
 283     recurse()
 284     columns {
 285         status()
 286         weather()
 287         name()
 288         lastSuccess()
 289         lastFailure()
 290         lastDuration()
 291         buildButton()
 292     }
 293 }
 294
 295 // PeriodicJobView: include all jobs that execute on a schedule
 296 Views.PeriodicJobView = listView('Periodic') {
 297     recurse()
 298     columns {
 299         status()
 300         weather()
 301         name()
 302         lastSuccess()
 303         lastFailure()
 304         lastDuration()
 305         buildButton()
 306     }
 307 }
 308
 309 // Create a view for non-PR jobs for each architecture.
 310 Constants.architectureList.each { architecture ->
 311     Views.ArchitectureViews[architecture] = listView(architecture) {
 312         recurse()
 313         columns {
 314             status()
 315             weather()
 316             name()
 317             lastSuccess()
 318             lastFailure()
 319             lastDuration()
 320             buildButton()
 321         }
 322     }
 323 }
 324
 325 // Create a view for non-PR jobs for each OS.
 326 Constants.osList.each { os ->
 327     // Don't create one for the special 'Windows_NT_BuildOnly'
 328     if (os == 'Windows_NT_BuildOnly') {
 329         return
 330     }
 331     Views.OSViews[os] = listView(os) {
 332         recurse()
 333         columns {
 334             status()
 335             weather()
 336             name()
 337             lastSuccess()
 338             lastFailure()
 339             lastDuration()
 340             buildButton()
 341         }
 342     }
 343 }
 344
 345 def static addToMergeView(def job) {
 346     Views.MergeJobView.with {
 347         jobs {
 348             name(job.name)
 349         }
 350     }
 351 }
 352
 353 def static addToPeriodicView(def job) {
 354     Views.PeriodicJobView.with {
 355         jobs {
 356             name(job.name)
 357         }
 358     }
 359 }
 360
 361 def static addToViews(def job, def isPR, def architecture, def os) {
 362     if (isPR) {
 363         // No views want PR jobs currently.
 364         return
 365     }
 366
 367     // Add to architecture view.
 368     Views.ArchitectureViews[architecture].with {
 369         jobs {
 370             name(job.name)
 371         }
 372     }
 373
 374     // Add to OS view.
 375     Views.OSViews[os].with {
 376         jobs {
 377             name(job.name)
 378         }
 379     }
 380 }
 381
 382 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 383     addToPeriodicView(job)
 384     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 385 }
 386
 387 def static addGithubPushTriggerHelper(def job) {
 388     addToMergeView(job)
 389     Utilities.addGithubPushTrigger(job)
 390 }
 391
 392
 393 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 394     assert os instanceof String
 395     assert architecture instanceof String
 396
 397     def armArches = ['arm', 'armlb', 'arm64']
 398     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 399
 400     if (!(architecture in armArches)) {
 401         assert options == null
 402         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 403
 404         return
 405     }
 406
 407     // This is an arm(64) job.
 408     //
 409     // There are several options.
 410     //
 411     // Windows_NT
 412     //
 413     // Arm32 (Build) -> latest-arm64
 414     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 415     // Arm32 (Test)  -> arm64-windows_nt
 416     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 417     //
 418     // Arm64 (Build) -> latest-arm64
 419     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 420     // Arm64 (Test)  -> arm64-windows_nt
 421     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 422     //
 423     // Ubuntu
 424     //
 425     // Arm32 (Build) -> arm-cross-latest
 426     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 427     // Arm32 (Test)  -> NYI Arch not supported
 428     //       |->
 429     //
 430     // Arm64 (Build) -> arm64-cross-latest
 431     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 432     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 433     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 434     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 435     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 436
 437     // This has to be a arm arch
 438     assert architecture in armArches
 439     if (os == "Windows_NT") {
 440         // Arm(64) Windows jobs share the same machines for now
 441         def isBuild = options['use_arm64_build_machine'] == true
 442
 443         if (isBuild == true) {
 444             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 445         } else {
 446             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 447         }
 448     } else {
 449         assert os != 'Windows_NT'
 450         assert os in supportedArmLinuxOs
 451
 452         if (architecture == 'arm' || architecture == 'armlb') {
 453             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 454         } else {
 455             // Arm64 Linux
 456             if (options['is_build_only'] == true) {
 457                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 458             } else {
 459                 // Arm64 Test Machines
 460                 if (options['large_pages'] == false) {
 461                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 462                 } else {
 463                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 464                 }
 465             }
 466         }
 467     }
 468 }
 469
 470 def static isGCStressRelatedTesting(def scenario) {
 471     // The 'r2r_gcstress15' scenario is a basic scenario.
 472     // Detect it and make it a GCStress related.
 473     if (scenario == 'r2r_gcstress15')
 474     {
 475         return true;
 476     }
 477
 478     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 479     def scenarioName = scenario.toLowerCase()
 480     def isGCStressTesting = false
 481     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 482         if (k in gcStressTestEnvVars) {
 483             isGCStressTesting = true;
 484         }
 485     }
 486     return isGCStressTesting
 487 }
 488
 489 def static isCoreFxScenario(def scenario) {
 490     def corefx_prefix = 'corefx_'
 491     if (scenario.length() < corefx_prefix.length()) {
 492         return false
 493     }
 494     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 495 }
 496
 497 def static isR2RBaselineScenario(def scenario) {
 498     return (scenario == 'r2r')
 499 }
 500
 501 def static isR2RStressScenario(def scenario) {
 502     return Constants.r2rStressScenarios.containsKey(scenario)
 503 }
 504
 505 def static isR2RScenario(def scenario) {
 506     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 507 }
 508
 509 def static isJitStressScenario(def scenario) {
 510     return Constants.jitStressModeScenarios.containsKey(scenario)
 511 }
 512
 513 def static isLongGc(def scenario) {
 514     return (scenario == 'longgc' || scenario == 'gcsimulator')
 515 }
 516
 517 def static isJitDiff(def scenario) {
 518     return (scenario == 'jitdiff')
 519 }
 520
 521 def static isGcReliabilityFramework(def scenario) {
 522     return (scenario == 'gc_reliability_framework')
 523 }
 524
 525 def static isArmWindowsScenario(def scenario) {
 526     return Constants.validArmWindowsScenarios.containsKey(scenario)
 527 }
 528
 529 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 530     // 2 hours (120 minutes) is the default timeout
 531     def timeout = 120
 532
 533     if (!(scenario == 'default' && isPR == true)) {
 534         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 535         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 536         timeout = 240
 537     }
 538
 539     if (!isBuildOnly) {
 540         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 541         if (isGCStressRelatedTesting(scenario)) {
 542             timeout = 4320
 543         }
 544         else if (isCoreFxScenario(scenario)) {
 545             timeout = 360
 546         }
 547         else if (isJitStressScenario(scenario)) {
 548             timeout = 240
 549         }
 550         else if (isR2RBaselineScenario(scenario)) {
 551             timeout = 240
 552         }
 553         else if (isLongGc(scenario)) {
 554             timeout = 1440
 555         }
 556         else if (isJitDiff(scenario)) {
 557             timeout = 240
 558         }
 559         else if (isGcReliabilityFramework(scenario)) {
 560             timeout = 1440
 561         }
 562         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 563             timeout = 240
 564         }
 565     }
 566
 567     if (configuration == 'Debug') {
 568         // Debug runs can be very slow. Add an hour.
 569         timeout += 60
 570     }
 571
 572     // If we've changed the timeout from the default, set it in the job.
 573
 574     if (timeout != 120) {
 575         Utilities.setJobTimeout(newJob, timeout)
 576     }
 577 }
 578
 579 def static getJobFolder(def scenario) {
 580     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 581         return 'jitstress'
 582     }
 583     if (scenario == 'illink') {
 584         return 'illink'
 585     }
 586     return ''
 587 }
 588
 589 def static getStressModeDisplayName(def scenario) {
 590     def displayStr = ''
 591     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 592         def prefixLength = 'COMPlus_'.length()
 593         if (k.length() >= prefixLength) {
 594             def modeName = k.substring(prefixLength, k.length())
 595             displayStr += ' ' + modeName + '=' + v
 596         }
 597     }
 598
 599     if (isCoreFxScenario(scenario)) {
 600         displayStr = ('CoreFx ' + displayStr).trim()
 601     }
 602
 603     return displayStr
 604 }
 605
 606 def static getR2RDisplayName(def scenario) {
 607     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 608     def displayStr = scenario
 609     def prefixLength = 'r2r_'.length()
 610     if (displayStr.length() >= prefixLength) {
 611         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 612     } else if (scenario == 'r2r') {
 613         displayStr = "R2R"
 614     }
 615     return displayStr
 616 }
 617
 618 //
 619 // Functions to create an environment script.
 620 //      envScriptCreate -- initialize the script (call first)
 621 //      envScriptFinalize -- finalize the script (call last)
 622 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 623 //      envScriptAppendExistingScript -- append an existing script to the generated script
 624 //
 625 // Each script returns a string of commands. Concatenate all the strings together before
 626 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 627 //
 628
 629 // Initialize the environment setting script.
 630 def static envScriptCreate(def os, def stepScriptLocation) {
 631     def stepScript = ''
 632     if (os == 'Windows_NT') {
 633         stepScript += "echo Creating TestEnv script\r\n"
 634         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 635
 636         // Create at least an empty script.
 637         stepScript += "echo. > ${stepScriptLocation}\r\n"
 638     }
 639     else {
 640         stepScript += "echo Creating environment setting script\n"
 641         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 642     }
 643
 644     return stepScript
 645 }
 646
 647 // Generates the string for setting stress mode variables.
 648 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 649     def stepScript = ''
 650     if (os == 'Windows_NT') {
 651         stressModeVars.each{ k, v ->
 652             // Write out what we are writing to the script file
 653             stepScript += "echo Setting ${k}=${v}\r\n"
 654             // Write out the set itself to the script file`
 655             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 656         }
 657     }
 658     else {
 659         stressModeVars.each{ k, v ->
 660             // Write out what we are writing to the script file
 661             stepScript += "echo Setting ${k}=${v}\n"
 662             // Write out the set itself to the script file`
 663             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 664         }
 665     }
 666
 667     return stepScript
 668 }
 669
 670 // Append an existing script to an environment script.
 671 // Returns string of commands to do this.
 672 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 673     assert (os == 'Windows_NT')
 674     def stepScript = ''
 675
 676     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 677     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 678
 679     return stepScript
 680 }
 681
 682 // Finalize an environment setting script.
 683 // Returns string of commands to do this.
 684 def static envScriptFinalize(def os, def stepScriptLocation) {
 685     def stepScript = ''
 686
 687     if (os == 'Windows_NT') {
 688         // Display the resulting script. This is useful when looking at the output log file.
 689         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 690         stepScript += "type ${stepScriptLocation}\r\n"
 691     }
 692     else {
 693         stepScript += "chmod +x ${stepScriptLocation}\n"
 694     }
 695
 696     return stepScript
 697 }
 698
 699 def static isNeedDocker(def architecture, def os, def isBuild) {
 700     if (isBuild) {
 701         if (architecture == 'x86' && os == 'Ubuntu') {
 702             return true
 703         }
 704         else if (architecture == 'arm') {
 705             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 706                 return true
 707             }
 708         }
 709     }
 710     else {
 711         if (architecture == 'x86' && os == 'Ubuntu') {
 712             return true
 713         }
 714     }
 715     return false
 716 }
 717
 718 def static getDockerImageName(def architecture, def os, def isBuild) {
 719     // We must change some docker private images to official later
 720     if (isBuild) {
 721         if (architecture == 'x86' && os == 'Ubuntu') {
 722             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 723         }
 724         else if (architecture == 'arm') {
 725             if (os == 'Ubuntu') {
 726                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 727             }
 728             else if (os == 'Ubuntu16.04') {
 729                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 730             }
 731             else if (os == 'Tizen') {
 732                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 733             }
 734         }
 735     }
 736     else {
 737         if (architecture == 'x86' && os == 'Ubuntu') {
 738             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 739         }
 740     }
 741     println("Unknown architecture to use docker: ${architecture} ${os}");
 742     assert false
 743 }
 744
 745 // Calculates the name of the build job based on some typical parameters.
 746 //
 747 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 748     // If the architecture is x64, do not add that info into the build name.
 749     // Need to change around some systems and other builds to pick up the right builds
 750     // to do that.
 751
 752     def suffix = scenario != 'default' ? "_${scenario}" : '';
 753     if (isBuildOnly) {
 754         suffix += '_bld'
 755     }
 756     def baseName = ''
 757     switch (architecture) {
 758         case 'x64':
 759             if (scenario == 'default') {
 760                 // For now we leave x64 off of the name for compatibility with other jobs
 761                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 762             }
 763             else if (scenario == 'formatting') {
 764                 // we don't care about the configuration for the formatting job. It runs all configs
 765                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 766             }
 767             else {
 768                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 769             }
 770             break
 771         case 'arm64':
 772             if (os.toLowerCase() == "windows_nt") {
 773                 // These are cross builds
 774                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 775             }
 776             else {
 777                 // Defaults to a small page size set of machines.
 778                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 779             }
 780             break
 781         case 'arm':
 782             // These are cross builds
 783             if (os == 'Tizen') {
 784                 // ABI: softfp
 785                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 786             }
 787             else {
 788                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 789             }
 790             break
 791         case 'armlb':
 792             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 793             break
 794         case 'x86':
 795         case 'x86_arm_altjit':
 796         case 'x64_arm64_altjit':
 797             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 798             break
 799         default:
 800             println("Unknown architecture: ${architecture}");
 801             assert false
 802             break
 803     }
 804
 805     return baseName + suffix
 806 }
 807
 808 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 809
 810     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 811     if (os == 'Windows_NT') {
 812         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 813             if (!(branch in Constants.WindowsArm64Branches)) {
 814                 return
 815             }
 816         }
 817     }
 818
 819     // Check scenario.
 820     switch (scenario) {
 821         case 'default':
 822             switch (architecture) {
 823                 case 'x64':
 824                 case 'x86':
 825                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 826                         addPeriodicTriggerHelper(job, '@daily')
 827                     }
 828                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 829                         addGithubPushTriggerHelper(job)
 830                     }
 831                     break
 832                 case 'arm':
 833                 case 'armlb':
 834                 case 'x86_arm_altjit':
 835                 case 'x64_arm64_altjit':
 836                     addGithubPushTriggerHelper(job)
 837                     break
 838                 case 'arm64':
 839                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 840                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 841                     break
 842                 default:
 843                     println("Unknown architecture: ${architecture}");
 844                     assert false
 845                     break
 846             }
 847             break
 848         case 'r2r':
 849             assert !(os in bidailyCrossList)
 850             // r2r gets a push trigger for checked/release
 851             if (configuration == 'Checked' || configuration == 'Release') {
 852                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 853                 if (architecture == 'x64' && os != 'OSX10.12') {
 854                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 855                     if (isFlowJob || os == 'Windows_NT') {
 856                         addGithubPushTriggerHelper(job)
 857                     }
 858                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 859                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 860                     if (isFlowJob) {
 861                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 862                     }
 863                 }
 864                 // For x86, only add per-commit jobs for Windows
 865                 else if (architecture == 'x86') {
 866                     if (os == 'Windows_NT') {
 867                         addGithubPushTriggerHelper(job)
 868                     }
 869                 }
 870                 // arm64 r2r jobs should only run daily.
 871                 else if (architecture == 'arm64') {
 872                     if (os == 'Windows_NT') {
 873                         addPeriodicTriggerHelper(job, '@daily')
 874                     }
 875                 }
 876             }
 877             break
 878         case 'r2r_jitstress1':
 879         case 'r2r_jitstress2':
 880         case 'r2r_jitstressregs1':
 881         case 'r2r_jitstressregs2':
 882         case 'r2r_jitstressregs3':
 883         case 'r2r_jitstressregs4':
 884         case 'r2r_jitstressregs8':
 885         case 'r2r_jitstressregs0x10':
 886         case 'r2r_jitstressregs0x80':
 887         case 'r2r_jitstressregs0x1000':
 888         case 'r2r_jitminopts':
 889         case 'r2r_jitforcerelocs':
 890         case 'r2r_gcstress15':
 891             assert !(os in bidailyCrossList)
 892
 893             // GCStress=C is currently not supported on OS X
 894             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 895                 break
 896             }
 897
 898             // GC Stress 15 r2r gets a push trigger for checked/release
 899             if (configuration == 'Checked' || configuration == 'Release') {
 900                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 901                 if (architecture == 'x64') {
 902                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 903                     if (isFlowJob || os == 'Windows_NT') {
 904                         // Add a weekly periodic trigger
 905                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 906                     }
 907                 }
 908                 // For x86, only add per-commit jobs for Windows
 909                 else if (architecture == 'x86') {
 910                     if (os == 'Windows_NT') {
 911                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 912                     }
 913                 }
 914             }
 915             break
 916         case 'longgc':
 917             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 918             assert configuration == 'Release'
 919             assert architecture == 'x64'
 920             addPeriodicTriggerHelper(job, '@daily')
 921             // TODO: Add once external email sending is available again
 922             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 923             break
 924         case 'gcsimulator':
 925             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 926             assert configuration == 'Release'
 927             assert architecture == 'x64'
 928             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 929             // TODO: Add once external email sending is available again
 930             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 931             break
 932         case 'standalone_gc':
 933             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 934             assert (configuration == 'Release' || configuration == 'Checked')
 935             // TODO: Add once external email sending is available again
 936             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 937             addPeriodicTriggerHelper(job, '@daily')
 938             break
 939         case 'gc_reliability_framework':
 940             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 941             assert (configuration == 'Release' || configuration == 'Checked')
 942             // Only triggered by phrase.
 943             break
 944         case 'ilrt':
 945             assert !(os in bidailyCrossList)
 946             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 947             if (architecture == 'x64' && configuration == 'Release') {
 948                 // We don't expect to see a job generated except in these scenarios
 949                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 950                 if (isFlowJob || os == 'Windows_NT') {
 951                     addPeriodicTriggerHelper(job, '@daily')
 952                 }
 953             }
 954             break
 955         case 'jitdiff':
 956             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 957             assert configuration == 'Checked'
 958             assert (architecture == 'x64' || architecture == 'x86')
 959             addGithubPushTriggerHelper(job)
 960             break
 961         case 'formatting':
 962             assert (os == 'Windows_NT' || os == "Ubuntu")
 963             assert architecture == 'x64'
 964             addGithubPushTriggerHelper(job)
 965             break
 966         case 'jitstressregs1':
 967         case 'jitstressregs2':
 968         case 'jitstressregs3':
 969         case 'jitstressregs4':
 970         case 'jitstressregs8':
 971         case 'jitstressregs0x10':
 972         case 'jitstressregs0x80':
 973         case 'jitstressregs0x1000':
 974         case 'minopts':
 975         case 'forcerelocs':
 976         case 'jitstress1':
 977         case 'jitstress2':
 978         case 'jitstress2_jitstressregs1':
 979         case 'jitstress2_jitstressregs2':
 980         case 'jitstress2_jitstressregs3':
 981         case 'jitstress2_jitstressregs4':
 982         case 'jitstress2_jitstressregs8':
 983         case 'jitstress2_jitstressregs0x10':
 984         case 'jitstress2_jitstressregs0x80':
 985         case 'jitstress2_jitstressregs0x1000':
 986         case 'tailcallstress':
 987         case 'jitsse2only':
 988         case 'jitnosimd':
 989         case 'jitnox86hwintrinsic':
 990         case 'jitincompletehwintrinsic':
 991         case 'jitx86hwintrinsicnoavx':
 992         case 'jitx86hwintrinsicnoavx2':
 993         case 'jitx86hwintrinsicnosimd':
 994         case 'corefx_baseline':
 995         case 'corefx_minopts':
 996         case 'corefx_jitstress1':
 997         case 'corefx_jitstress2':
 998         case 'corefx_jitstressregs1':
 999         case 'corefx_jitstressregs2':
1000         case 'corefx_jitstressregs3':
1001         case 'corefx_jitstressregs4':
1002         case 'corefx_jitstressregs8':
1003         case 'corefx_jitstressregs0x10':
1004         case 'corefx_jitstressregs0x80':
1005         case 'corefx_jitstressregs0x1000':
1006         case 'zapdisable':
1007             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1008                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1009                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1010                     if (os == 'Windows_NT') {
1011                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1012                         addPeriodicTriggerHelper(job, '@weekly')
1013                     }
1014                 }
1015                 else {
1016                     addPeriodicTriggerHelper(job, '@daily')
1017                 }
1018             }
1019             break
1020         case 'heapverify1':
1021         case 'gcstress0x3':
1022             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1023                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1024                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1025                     if (os == 'Windows_NT') {
1026                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1027                         addPeriodicTriggerHelper(job, '@weekly')
1028                     }
1029                     // TODO: Add once external email sending is available again
1030                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1031                 }
1032                 else {
1033                     addPeriodicTriggerHelper(job, '@weekly')
1034                 }
1035             }
1036             break
1037         case 'gcstress0xc':
1038         case 'gcstress0xc_zapdisable':
1039         case 'gcstress0xc_zapdisable_jitstress2':
1040         case 'gcstress0xc_zapdisable_heapverify1':
1041         case 'gcstress0xc_jitstress1':
1042         case 'gcstress0xc_jitstress2':
1043         case 'gcstress0xc_minopts_heapverify1':
1044             // GCStress=C is currently not supported on OS X
1045             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1046                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1047                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1048                     if (os == 'Windows_NT') {
1049                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1050                         addPeriodicTriggerHelper(job, '@weekly')
1051                     }
1052                     // TODO: Add once external email sending is available again
1053                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1054                 }
1055                 else {
1056                     addPeriodicTriggerHelper(job, '@weekly')
1057                 }
1058             }
1059             break
1060
1061         case 'illink':
1062             // Testing on other operating systems TBD
1063             assert (os == 'Windows_NT' || os == 'Ubuntu')
1064             if (architecture == 'x64' || architecture == 'x86') {
1065                 if (configuration == 'Checked') {
1066                     addPeriodicTriggerHelper(job, '@daily')
1067                 }
1068             }
1069             break
1070
1071         case 'tieredcompilation':
1072         case 'corefx_tieredcompilation':
1073             // No periodic jobs just yet, still testing
1074             break
1075
1076         default:
1077             println("Unknown scenario: ${scenario}");
1078             assert false
1079             break
1080     }
1081     return
1082 }
1083
1084 // **************************
1085 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1086 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1087 // builds will do a couple extra steps.
1088 // **************************
1089
1090 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1091 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1092 // it's just used for internal builds
1093 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1094 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1095     if (isWindowsBuildOnlyJob) {
1096         return
1097     }
1098
1099     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1100     // Non pull request builds.
1101     if (!isPR) {
1102         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1103         return
1104     }
1105
1106      def arm64Users = [
1107         'AndyAyersMS',
1108         'briansull',
1109         'BruceForstall',
1110         'CarolEidt',
1111         'cmckinsey',
1112         'erozenfeld',
1113         'janvorli',
1114         'jashook',
1115         'JosephTremoulet',
1116         'pgodeq',
1117         'pgavlin',
1118         'rartemev',
1119         'russellhadley',
1120         'RussKeldorph',
1121         'sandreenko',
1122         'sdmaclea',
1123         'swaroop-sridhar',
1124         'jkotas',
1125         'markwilkie',
1126         'weshaggard'
1127     ]
1128
1129     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1130     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1131     def osGroup = getOSGroup(os)
1132     switch (architecture) {
1133         case 'x64': // editor brace matching: {
1134             if (scenario == 'formatting') {
1135                 assert configuration == 'Checked'
1136                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1137                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1138                 }
1139                 break
1140             }
1141
1142             switch (os) {
1143                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1144                 case 'Debian8.4':
1145                 case 'RHEL7.2':
1146                     if (scenario == 'default') {
1147                         assert !isFlowJob
1148                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1149                     }
1150                     break
1151                 case 'Ubuntu16.04':
1152                     assert !isFlowJob
1153                     assert scenario == 'default'
1154                     // Distinguish with the other architectures (arm and x86)
1155                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1156                     break
1157                 case 'Fedora24':
1158                 case 'Ubuntu16.10':
1159                     assert !isFlowJob
1160                     assert scenario == 'default'
1161                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1162                     break
1163                 case 'Ubuntu':
1164                     if (scenario == 'illink') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1166                         break
1167                     }
1168                     // fall through
1169                 case 'OSX10.12':
1170                     // Triggers on the non-flow jobs aren't necessary here
1171                     // Corefx testing uses non-flow jobs.
1172                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1173                         break
1174                     }
1175                     switch (scenario) {
1176                         case 'default':
1177                             // OSX uses checked for default PR tests
1178                             if (configuration == 'Checked') {
1179                                 // Default trigger
1180                                 assert !job.name.contains("centos")
1181                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1182                             }
1183                             break
1184                         case 'jitdiff':
1185                             if (configuration == 'Checked') {
1186                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1187                             }
1188                             break
1189                         case 'ilrt':
1190                             if (configuration == 'Release') {
1191                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1192                             }
1193                             break
1194                         case 'longgc':
1195                             if (configuration == 'Release') {
1196                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1197                             }
1198                             break
1199                         case 'gcsimulator':
1200                             if (configuration == 'Release') {
1201                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1202                             }
1203                             break
1204                         case 'standalone_gc':
1205                             if (configuration == 'Release' || configuration == 'Checked') {
1206                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1207                             }
1208                             break
1209                         case 'gc_reliability_framework':
1210                             if (configuration == 'Release' || configuration == 'Checked') {
1211                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1212                             }
1213                             break
1214                         default:
1215                             if (isJitStressScenario(scenario)) {
1216                                 def displayStr = getStressModeDisplayName(scenario)
1217                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1218                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1219                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1220                             }
1221                             else if (isR2RScenario(scenario)) {
1222                                 if (configuration == 'Release' || configuration == 'Checked') {
1223                                     def displayStr = getR2RDisplayName(scenario)
1224                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1225                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1226                                 }
1227                             }
1228                             else {
1229                                 println("Unknown scenario: ${scenario}");
1230                                 assert false
1231                             }
1232                             break
1233                     }
1234                     break
1235
1236                 case 'CentOS7.1':
1237                     switch (scenario) {
1238                         case 'default':
1239                             // CentOS uses checked for default PR tests while debug is build only
1240                             if (configuration == 'Debug') {
1241                                 // Default trigger
1242                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1243                             }
1244
1245                             // Make sure this is a flow job to get build and test.
1246                             if (configuration == 'Checked' && isFlowJob) {
1247                                 assert job.name.contains("flow")
1248                                 // Default trigger
1249                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1250                             }
1251                             break
1252                         default:
1253                             if (isR2RScenario(scenario)) {
1254                                 if (configuration == 'Release' || configuration == 'Checked') {
1255                                     def displayStr = getR2RDisplayName(scenario)
1256                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1257                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                                 }
1259                             }
1260                             break
1261                     }
1262                     break
1263
1264                 case 'Windows_NT':
1265                     switch (scenario) {
1266                         case 'default':
1267                             // Default trigger
1268                             if (configuration == 'Checked') {
1269                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1270                             }
1271                             break
1272                         case 'jitdiff':
1273                             if (configuration == 'Checked') {
1274                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1275                             }
1276                             break
1277                         case 'ilrt':
1278                             if (configuration == 'Release') {
1279                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1280                             }
1281                             break
1282                         case 'longgc':
1283                             if (configuration == 'Release') {
1284                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1285                             }
1286                             break
1287                         case 'gcsimulator':
1288                             if (configuration == 'Release') {
1289                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1290                             }
1291                             break
1292                         case 'standalone_gc':
1293                             if (configuration == 'Release' || configuration == 'Checked') {
1294                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1295                             }
1296                             break
1297                         case 'gc_reliability_framework':
1298                             if (configuration == 'Release' || configuration == 'Checked') {
1299                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1300                             }
1301                             break
1302                         case 'illink':
1303                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1304                             break
1305                         default:
1306                             if (isJitStressScenario(scenario)) {
1307                                 def displayStr = getStressModeDisplayName(scenario)
1308                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1309                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1310                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1311                             }
1312                             else if (isR2RScenario(scenario)) {
1313                                 if (configuration == 'Release' || configuration == 'Checked') {
1314                                     def displayStr = getR2RDisplayName(scenario)
1315                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1316                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1317                                 }
1318                             }
1319                             else {
1320                                 println("Unknown scenario: ${scenario}");
1321                                 assert false
1322                             }
1323                             break
1324                     }
1325                     break
1326                 default:
1327                     println("Unknown os: ${os}");
1328                     assert false
1329                     break
1330             }
1331             break
1332         // editor brace matching: }
1333         case 'armlb':
1334         case 'arm': // editor brace matching: {
1335             switch (os) {
1336                 case 'Ubuntu':
1337                 case 'Ubuntu16.04':
1338                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1339                         break
1340                     }
1341                     assert scenario == 'default'
1342                     job.with {
1343                         publishers {
1344                             azureVMAgentPostBuildAction {
1345                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1346                             }
1347                         }
1348                     }
1349                     if (configuration == 'Debug') {
1350                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1351                     }
1352                     else {
1353                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1354                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1355                     }
1356                     break
1357                 case 'Tizen':
1358                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1359                         break
1360                     }
1361                     architecture='armel'
1362                     job.with {
1363                         publishers {
1364                             azureVMAgentPostBuildAction {
1365                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1366                             }
1367                         }
1368                     }
1369                     if (configuration == 'Checked') {
1370                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1371                     }
1372                     else {
1373                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1374                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1375                     }
1376                     break
1377                 case 'Windows_NT':
1378                     // Triggers on the non-flow jobs aren't necessary here
1379                     if (!isFlowJob) {
1380                         break
1381                     }
1382
1383                     // Set up a private trigger
1384                     def contextString = "${os} ${architecture} Cross ${configuration}"
1385                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1386                     if (scenario == 'default') {
1387                         contextString += " Innerloop"
1388                         triggerString += "\\W+Innerloop"
1389                     }
1390                     else {
1391                         contextString += " ${scenario}"
1392                         triggerString += "\\W+${scenario}"
1393                     }
1394
1395                     if (configuration == 'Debug') {
1396                         contextString += " Build"
1397                         triggerString += "\\W+Build"
1398                     } else {
1399                         contextString += " Build and Test"
1400                         triggerString += "\\W+Build and Test"
1401                     }
1402
1403                     triggerString += ".*"
1404
1405                     switch (scenario) {
1406                         case 'default':
1407                             // Only Checked is a default trigger.
1408                             if (configuration == 'Checked')
1409                             {
1410                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1411                             }
1412                             else
1413                             {
1414                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1415                             }
1416                             break
1417                         default:
1418                             // Stress jobs will use this code path.
1419                             if (isArmWindowsScenario(scenario)) {
1420                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1421                             }
1422                             break
1423                     }
1424                     break
1425                 default:
1426                     println("NYI os: ${os}");
1427                     assert false
1428                     break
1429             }
1430             break
1431         // editor brace matching: }
1432         case 'arm64': // editor brace matching: {
1433             // Set up a private trigger
1434             def contextString = "${os} ${architecture} Cross ${configuration}"
1435             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1436             if (scenario == 'default') {
1437                 contextString += " Innerloop"
1438                 triggerString += "\\W+Innerloop"
1439             }
1440             else {
1441                 contextString += " ${scenario}"
1442                 triggerString += "\\W+${scenario}"
1443             }
1444
1445             if (configuration == 'Debug') {
1446                 contextString += " Build"
1447                 triggerString += "\\W+Build"
1448             } else {
1449                 contextString += " Build and Test"
1450                 triggerString += "\\W+Build and Test"
1451             }
1452
1453             triggerString += ".*"
1454
1455             switch (os) {
1456                 case 'Ubuntu':
1457                 case 'Ubuntu16.04':
1458                     switch (scenario) {
1459                         case 'default':
1460                             if (configuration == 'Debug' && !isFlowJob) {
1461                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1462                             }
1463                             else {
1464                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1465                             }
1466
1467                             break
1468                         default:
1469                             if (isR2RScenario(scenario)) {
1470                                 if (configuration == 'Checked' || configuration == 'Release') {
1471                                     def displayStr = getR2RDisplayName(scenario)
1472                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1473                                 }
1474                             }
1475                             break
1476                     }
1477                     break
1478                 case 'Windows_NT':
1479                     // Triggers on the non-flow jobs aren't necessary here
1480                     if (!isFlowJob) {
1481                         break
1482                     }
1483
1484                     assert isArmWindowsScenario(scenario)
1485                     switch (scenario) {
1486                         case 'default':
1487                             if (configuration == 'Checked') {
1488                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1489                             }
1490                             else {
1491                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1492                             }
1493                             break
1494                         default:
1495                             // Stress jobs will use this code path.
1496                             if (isArmWindowsScenario(scenario)) {
1497                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1498                             }
1499                             break
1500                     }
1501                     break
1502                 default:
1503                     println("NYI os: ${os}");
1504                     assert false
1505                     break
1506             }
1507             break
1508         // editor brace matching: }
1509         case 'x86': // editor brace matching: {
1510             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1511             if (os == 'Ubuntu') {
1512                 // Triggers on the non-flow jobs aren't necessary here
1513                 if (!isFlowJob) {
1514                     break
1515                 }
1516                 // on-demand only for ubuntu x86
1517                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1518                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1519                 break
1520             }
1521             switch (scenario) {
1522                 case 'default':
1523                     if (configuration == 'Checked') {
1524                         assert !job.name.contains("centos")
1525                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1526                     }
1527                     else {
1528                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1529                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1530                     }
1531                     break
1532                 case 'ilrt':
1533                     if (configuration == 'Release') {
1534                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1535                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1536                     }
1537                     break
1538                 case 'longgc':
1539                     if (configuration == 'Release') {
1540                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1541                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1542                     }
1543                     break
1544                 case 'gcsimulator':
1545                     if (configuration == 'Release') {
1546                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1547                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1548                     }
1549                     break
1550                 case 'standalone_gc':
1551                     if (configuration == 'Release' || configuration == 'Checked') {
1552                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1553                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1554                     }
1555                     break
1556                 case 'illink':
1557                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1558                     break
1559                 default:
1560                     if (isJitStressScenario(scenario)) {
1561                         def displayStr = getStressModeDisplayName(scenario)
1562                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1563                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1564                     }
1565                     else if (isR2RScenario(scenario)) {
1566                         if (configuration == 'Release' || configuration == 'Checked') {
1567                             def displayStr = getR2RDisplayName(scenario)
1568                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1569                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1570                         }
1571                     }
1572                     else {
1573                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1574                         assert false
1575                     }
1576                     break
1577             }
1578             break
1579          // editor brace matching: }
1580         case 'x64_arm64_altjit':
1581         case 'x86_arm_altjit': // editor brace matching: {
1582             assert (os == 'Windows_NT')
1583             switch (scenario) {
1584                 case 'default':
1585                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1586                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1587                     break
1588                 default:
1589                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1590                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1591                     break
1592             }
1593             break
1594         // editor brace matching: }
1595         default:
1596             println("Unknown architecture: ${architecture}");
1597             assert false
1598             break
1599     }
1600 }
1601
1602 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1603     def buildCommands = [];
1604     def osGroup = getOSGroup(os)
1605     def lowerConfiguration = configuration.toLowerCase()
1606
1607     def priority = '1'
1608     if (scenario == 'default' && isPR == true) {
1609         priority = '0'
1610     }
1611
1612     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1613
1614     def enableCorefxTesting = isCoreFxScenario(scenario)
1615
1616     // Calculate the build steps, archival, and xunit results
1617     switch (os) {
1618         case 'Windows_NT': // editor brace matching: {
1619             switch (architecture) {
1620                 case 'x64':
1621                 case 'x86':
1622                 case 'x86_arm_altjit':
1623                 case 'x64_arm64_altjit':
1624                     def arch = architecture
1625                     def buildOpts = ''
1626                     if (architecture == 'x86_arm_altjit') {
1627                         arch = 'x86'
1628                     }
1629                     else if (architecture == 'x64_arm64_altjit') {
1630                         arch = 'x64'
1631                     }
1632
1633                     if (scenario == 'formatting') {
1634                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1635                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1636                         break
1637                     }
1638
1639                     if (scenario == 'illink') {
1640                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1641                     }
1642
1643                     // If it is a release build for windows, ensure PGO is used, else fail the build
1644                     if ((lowerConfiguration == 'release') &&
1645                         (scenario in Constants.basicScenarios) &&
1646                         (architecture != 'x86_arm_altjit') &&
1647                         (architecture != 'x64_arm64_altjit')) {
1648
1649                         buildOpts += ' -enforcepgo'
1650                     }
1651
1652                     if (enableCorefxTesting) {
1653                         buildOpts += ' skiptests';
1654                     } else {
1655                         buildOpts += " -priority=${priority}"
1656                     }
1657
1658                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1659                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1660                     // 35 characters long.
1661
1662                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1663
1664                     if (!isBuildOnly) {
1665                         def runtestArguments = ''
1666                         def testOpts = 'collectdumps'
1667
1668                         if (isR2RScenario(scenario)) {
1669
1670                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1671                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1672                             // to cause the tests to be crossgen'ed.
1673
1674                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1675                                 testOpts += ' crossgenaltjit protononjit.dll'
1676                             } else {
1677                                 testOpts += ' crossgen'
1678                             }
1679
1680                             testOpts += ' runcrossgentests'
1681                         }
1682                         else if (scenario == 'jitdiff') {
1683                             testOpts += ' jitdisasm crossgen'
1684                         }
1685                         else if (scenario == 'ilrt') {
1686                             testOpts += ' ilasmroundtrip'
1687                         }
1688                         else if (isLongGc(scenario)) {
1689                             testOpts += " ${scenario} sequential"
1690                         }
1691                         else if (scenario == 'standalone_gc') {
1692                             testOpts += ' gcname clrgc.dll'
1693                         }
1694                         else if (scenario == 'illink') {
1695                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1696                         }
1697
1698                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1699                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1700                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1701                         // which is read by the xunit harness.)
1702                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1703                         {
1704                             def timeout = 1800000
1705                             testOpts += " timeout ${timeout}"
1706                         }
1707
1708                         // If we are running a stress mode, we should write out the set of key
1709                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1710
1711                         def envScriptPath = ''
1712                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1713                             def buildCommandsStr = ''
1714                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1715                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1716
1717                             if (isJitStressScenario(scenario)) {
1718                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1719                             }
1720                             else if (isR2RStressScenario(scenario)) {
1721                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1722                             }
1723
1724                             if (architecture == 'x86_arm_altjit') {
1725                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1726                             }
1727                             else if (architecture == 'x64_arm64_altjit') {
1728                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1729                             }
1730
1731                             envScriptFinalize(os, envScriptPath)
1732
1733                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1734                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1735                             // into a single script.
1736                             buildCommands += buildCommandsStr
1737                         }
1738                         else if (architecture == 'x86_arm_altjit') {
1739                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1740                         }
1741                         else if (architecture == 'x64_arm64_altjit') {
1742                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1743                         }
1744                         if (envScriptPath != '') {
1745                             testOpts += " TestEnv ${envScriptPath}"
1746                         }
1747
1748                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1749
1750                         if (enableCorefxTesting) {
1751                             def workspaceRelativeFxRoot = "_/fx"
1752                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1753
1754                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1755
1756                             // Archive and process (only) the test results
1757                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1758                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1759
1760                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1761                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1762                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1763                         }
1764                         else if (isGcReliabilityFramework(scenario)) {
1765                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1766                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1767                         }
1768                         else {
1769                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1770                         }
1771                     }
1772
1773                     if (!enableCorefxTesting) {
1774                         // Run the rest of the build
1775                         // Build the mscorlib for the other OS's
1776                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1777                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1778
1779                         if (arch == "x64") {
1780                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1781                         }
1782
1783                         // Zip up the tests directory so that we don't use so much space/time copying
1784                         // 10s of thousands of files around.
1785                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1786
1787                         if (!isJitStressScenario(scenario)) {
1788                             // For windows, pull full test results and test drops for x86/x64.
1789                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1790                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1791                         }
1792
1793                         if (scenario == 'jitdiff') {
1794                             // retrive jit-dasm output for base commit, and run jit-diff
1795                             if (!isBuildOnly) {
1796                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1797                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1798                             }
1799                         }
1800
1801                         if (!isBuildOnly) {
1802                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1803                         }
1804                     }
1805                     break
1806                 case 'armlb':
1807                 case 'arm':
1808                     assert isArmWindowsScenario(scenario)
1809
1810                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1811                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1812
1813                     def buildArchitecture = 'arm'
1814
1815                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1816                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1817                     def armCrossgenOpt = ''
1818                     if (architecture == 'armlb') {
1819                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1820                     }
1821
1822                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1823                     priority = '1'
1824
1825                     // This is now a build only job. Do not run tests. Use the flow job.
1826                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1827
1828                     // Zip up the tests directory so that we don't use so much space/time copying
1829                     // 10s of thousands of files around.
1830                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1831
1832                     // Add archival.
1833                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1834                     break
1835                 case 'arm64':
1836                     assert isArmWindowsScenario(scenario)
1837
1838                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1839                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1840
1841                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1842                     priority = '1'
1843
1844                     // This is now a build only job. Do not run tests. Use the flow job.
1845                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1846
1847                     // Zip up the tests directory so that we don't use so much space/time copying
1848                     // 10s of thousands of files around.
1849                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1850
1851                     // Add archival.
1852                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1853                     break
1854                 default:
1855                     println("Unknown architecture: ${architecture}");
1856                     assert false
1857                     break
1858             }
1859             break
1860         // editor brace matching: }
1861         case 'Ubuntu':
1862         case 'Ubuntu16.04':
1863         case 'Ubuntu16.10':
1864         case 'Debian8.4':
1865         case 'OSX10.12':
1866         case 'CentOS7.1':
1867         case 'RHEL7.2':
1868         case 'Tizen':
1869         case 'Fedora24': // editor brace matching: {
1870             switch (architecture) {
1871                 case 'x64':
1872                 case 'x86':
1873                     if (architecture == 'x86' && os == 'Ubuntu') {
1874                         // build and PAL test
1875                         def dockerImage = getDockerImageName(architecture, os, true)
1876                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1877                         dockerImage = getDockerImageName(architecture, os, false)
1878                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1879                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1880                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1881                         break
1882                     }
1883
1884                     if (scenario == 'formatting') {
1885                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1886                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1887                         break
1888                     }
1889
1890                     if (scenario == 'illink') {
1891                         assert(os == 'Ubuntu')
1892                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1893                     }
1894
1895                     if (!enableCorefxTesting) {
1896                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1897                         // only on supported OS platforms.
1898                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1899                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1900
1901                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1902                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1903
1904                         // Basic archiving of the build
1905                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1906                         // And pal tests
1907                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1908                     }
1909                     else {
1910                         // Corefx stress testing
1911                         assert os == 'Ubuntu'
1912                         assert architecture == 'x64'
1913                         assert lowerConfiguration == 'checked'
1914                         assert isJitStressScenario(scenario)
1915
1916                         // Build coreclr
1917                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1918
1919                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1920
1921                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1922                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1923                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1924                         buildCommands += envScriptCmds
1925
1926                         // Build and text corefx
1927                         def workspaceRelativeFxRoot = "_/fx"
1928                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1929
1930                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1931
1932                         // Archive and process (only) the test results
1933                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1934                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1935                     }
1936                     break
1937                 case 'arm64':
1938                     if (!enableCorefxTesting) {
1939                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1940
1941                         // HACK -- Arm64 does not have corefx jobs yet.
1942                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1943                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1944                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1945                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1946
1947                         // Basic archiving of the build
1948                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1949                     }
1950                     break
1951                 case 'arm':
1952                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1953                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1954
1955                     // default values for Ubuntu
1956                     def arm_abi="arm"
1957                     def linuxCodeName="trusty"
1958                     if (os == 'Ubuntu16.04') {
1959                         linuxCodeName="xenial"
1960                     }
1961                     else if (os == 'Tizen') {
1962                         arm_abi="armel"
1963                         linuxCodeName="tizen"
1964                     }
1965
1966                     // Unzip the Windows test binaries first. Exit with 0
1967                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1968
1969                     // Unpack the corefx binaries
1970                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1971                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1972                     if (os != 'Tizen') {
1973                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1974                     }
1975                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1976                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1977                     if (os == 'Ubuntu') {
1978                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1979                     }
1980
1981                     // Call the ARM CI script to cross build and test using docker
1982                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1983                     --mode=docker \\
1984                     --${arm_abi} \\
1985                     --linuxCodeName=${linuxCodeName} \\
1986                     --buildConfig=${lowerConfiguration} \\
1987                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1988                     --coreFxBinDir=./bin/CoreFxBinDir \\
1989                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1990
1991                     // Basic archiving of the build, no pal tests
1992                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1993                     break
1994                 default:
1995                     println("Unknown architecture: ${architecture}");
1996                     assert false
1997                     break
1998             }
1999             break
2000         // editor brace matching: }
2001         default:
2002             println("Unknown os: ${os}");
2003             assert false
2004             break
2005     } // os
2006
2007     return buildCommands
2008 }
2009
2010 Constants.allScenarios.each { scenario ->
2011     [true, false].each { isPR ->
2012         Constants.architectureList.each { architecture ->
2013             Constants.configurationList.each { configuration ->
2014                 Constants.osList.each { os ->
2015                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2016                     // and reset the os to Windows_NT
2017                     def isBuildOnly = false
2018                     if (os == 'Windows_NT_BuildOnly') {
2019                         isBuildOnly = true
2020                         os = 'Windows_NT'
2021                     }
2022
2023                     // Tizen is only supported for arm architecture
2024                     if (os == 'Tizen' && architecture != 'arm') {
2025                         return
2026                     }
2027
2028                     // Skip totally unimplemented (in CI) configurations.
2029                     switch (architecture) {
2030                         case 'arm64':
2031                             if (os == 'Ubuntu16.04') {
2032                                 os = 'Ubuntu'
2033                             }
2034
2035                             // Windows and Ubuntu only
2036                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2037                                 return
2038                             }
2039                             break
2040                         case 'arm':
2041                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2042                                 return
2043                             }
2044                             break
2045                         case 'armlb':
2046                             if (os != 'Windows_NT') {
2047                                 return
2048                             }
2049                             break
2050                         case 'x86':
2051                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2052                                 return
2053                             }
2054                             break
2055                         case 'x86_arm_altjit':
2056                         case 'x64_arm64_altjit':
2057                             if (os != 'Windows_NT') {
2058                                 return
2059                             }
2060                             break
2061                         case 'x64':
2062                             // Everything implemented
2063                             break
2064                         default:
2065                             println("Unknown architecture: ${architecture}")
2066                             assert false
2067                             break
2068                     }
2069
2070                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2071                     // with checked builds)
2072                     if (isJitStressScenario(scenario)) {
2073                         if (configuration != 'Checked') {
2074                             return
2075                         }
2076
2077                         // Since these are just execution time differences,
2078                         // skip platforms that don't execute the tests here (Windows_NT only)
2079                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2080                         if (!isEnabledOS || isBuildOnly) {
2081                             return
2082                         }
2083
2084                         switch (architecture) {
2085                             case 'x64':
2086                             case 'x86':
2087                             case 'x86_arm_altjit':
2088                             case 'x64_arm64_altjit':
2089                                 // x86 ubuntu: default only
2090                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2091                                     return
2092                                 }
2093                                 // Windows: Everything implemented
2094                                 break
2095
2096                             default:
2097                                 // arm, arm64, armlb: stress is handled through flow jobs.
2098                                 return
2099                         }
2100                     }
2101                     else if (isR2RScenario(scenario)) {
2102                         if (os != 'Windows_NT') {
2103                             return
2104                         }
2105                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2106                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2107                             return
2108                         }
2109                     }
2110                     else {
2111                         // Skip scenarios
2112                         switch (scenario) {
2113                             case 'ilrt':
2114                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2115                                 // the default scenario build
2116                                 if (os != 'Windows_NT') {
2117                                     return
2118                                 }
2119                                 // Only x64 for now
2120                                 if (architecture != 'x64') {
2121                                     return
2122                                 }
2123                                 // Release only
2124                                 if (configuration != 'Release') {
2125                                     return
2126                                 }
2127                                 break
2128                             case 'jitdiff':
2129                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2130                                     return
2131                                 }
2132                                 if (architecture != 'x64') {
2133                                     return
2134                                 }
2135                                 if (configuration != 'Checked') {
2136                                     return
2137                                 }
2138                                 break
2139                             case 'longgc':
2140                             case 'gcsimulator':
2141                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2142                                     return
2143                                 }
2144                                 if (architecture != 'x64') {
2145                                     return
2146                                 }
2147                                 if (configuration != 'Release') {
2148                                     return
2149                                 }
2150                                 break
2151                             case 'gc_reliability_framework':
2152                             case 'standalone_gc':
2153                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2154                                     return
2155                                 }
2156
2157                                 if (architecture != 'x64') {
2158                                     return
2159                                 }
2160
2161                                 if (configuration != 'Release' && configuration != 'Checked') {
2162                                     return
2163                                 }
2164                                 break
2165                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2166                             case 'formatting':
2167                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2168                                     return
2169                                 }
2170                                 if (architecture != 'x64') {
2171                                     return
2172                                 }
2173                                 if (configuration != 'Checked') {
2174                                     return
2175                                 }
2176                                 if (isBuildOnly) {
2177                                     return
2178                                 }
2179                                 break
2180                             case 'illink':
2181                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2182                                     return
2183                                 }
2184                                 if (architecture != 'x64' && architecture != 'x86') {
2185                                     return
2186                                 }
2187                                 if (isBuildOnly) {
2188                                     return
2189                                 }
2190                                 break
2191                             case 'default':
2192                                 // Nothing skipped
2193                                 break
2194                             default:
2195                                 println("Unknown scenario: ${scenario}")
2196                                 assert false
2197                                 break
2198                         }
2199                     }
2200
2201                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2202                     // runtime behavior, not compile-time behavior, are not interesting.
2203                     switch (architecture) {
2204                         case 'x86_arm_altjit':
2205                         case 'x64_arm64_altjit':
2206                             if (isGCStressRelatedTesting(scenario)) {
2207                                 return
2208                             }
2209                             break
2210                         default:
2211                             break
2212                     }
2213
2214                     // Calculate names
2215                     def lowerConfiguration = configuration.toLowerCase()
2216                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2217                     def folderName = getJobFolder(scenario)
2218
2219                     // Create the new job
2220                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2221                     addToViews(newJob, isPR, architecture, os)
2222
2223                     def machineAffinityOptions = null
2224
2225                     if (os != 'Windows_NT') {
2226                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2227                     }
2228                     else {
2229                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2230                     }
2231
2232                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2233
2234                     // Add all the standard options
2235                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2236                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2237
2238                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2239                     def osGroup = getOSGroup(os)
2240
2241                     newJob.with {
2242                         steps {
2243                             if (os == 'Windows_NT') {
2244                                 buildCommands.each { buildCommand ->
2245                                     batchFile(buildCommand)
2246                                 }
2247                             }
2248                             else {
2249                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2250                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2251                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2252                                     // Define the Windows Tests and Corefx build job names
2253                                     def WindowsTestsName = projectFolder + '/' +
2254                                                            Utilities.getFullJobName(project,
2255                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2256                                                                                     false)
2257                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2258                                                        Utilities.getFolderName(branch)
2259
2260                                     // Copy the Windows test binaries and the Corefx build binaries
2261                                     copyArtifacts(WindowsTestsName) {
2262                                         includePatterns('bin/tests/tests.zip')
2263                                         buildSelector {
2264                                             latestSuccessful(true)
2265                                         }
2266                                     }
2267
2268                                     def arm_abi = 'arm'
2269                                     def corefx_os = 'linux'
2270                                     if (os == 'Tizen') {
2271                                         arm_abi = 'armel'
2272                                         corefx_os = 'tizen'
2273                                     }
2274
2275                                     // Let's use release CoreFX to test checked CoreCLR,
2276                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2277                                     def corefx_lowerConfiguration = lowerConfiguration
2278                                     if ( lowerConfiguration == 'checked' ) {
2279                                         corefx_lowerConfiguration='release'
2280                                     }
2281
2282                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2283                                         includePatterns('bin/build.tar.gz')
2284                                         buildSelector {
2285                                             latestSuccessful(true)
2286                                         }
2287                                     }
2288                                 }
2289
2290                                 buildCommands.each { buildCommand ->
2291                                     shell(buildCommand)
2292                                 }
2293                             }
2294                         }
2295                     } // newJob.with
2296
2297                 } // os
2298             } // configuration
2299         } // architecture
2300     } // isPR
2301 } // scenario
2302
2303
2304 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2305 Constants.allScenarios.each { scenario ->
2306     [true, false].each { isPR ->
2307         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2308             Constants.crossList.each { os ->
2309                 if (architecture == 'arm64') {
2310                     if (os != "Ubuntu" && os != "Windows_NT") {
2311                         return
2312                     }
2313                 } else if (architecture == 'arm' || architecture == 'armlb') {
2314                     if (os != 'Windows_NT') {
2315                         return
2316                     }
2317                 }
2318                 else if (architecture == 'x86') {
2319                     if (os != "Ubuntu") {
2320                         return
2321                     }
2322                 }
2323
2324                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2325
2326                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2327                     return
2328                 }
2329
2330                 Constants.configurationList.each { configuration ->
2331
2332                     // First, filter based on OS.
2333
2334                     if (os == 'Windows_NT') {
2335                         if (!isArmWindowsScenario(scenario)) {
2336                             return
2337                         }
2338                     }
2339                     else {
2340                         // Non-Windows
2341                         if (architecture == 'arm64') {
2342                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2343                                 return
2344                             }
2345                         }
2346                         else if (architecture == 'x86') {
2347                             // Linux/x86 only want default test
2348                             if (scenario != 'default') {
2349                                 return
2350                             }
2351                         }
2352                     }
2353
2354                     // For CentOS, we only want Checked/Release builds.
2355                     if (os == 'CentOS7.1') {
2356                         if (configuration != 'Checked' && configuration != 'Release') {
2357                             return
2358                         }
2359                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2360                             return
2361                         }
2362                     }
2363
2364                     // For RedHat and Debian, we only do Release builds.
2365                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2366                         if (configuration != 'Release') {
2367                             return
2368                         }
2369                         if (scenario != 'default') {
2370                             return
2371                         }
2372                     }
2373
2374                     // Next, filter based on scenario.
2375
2376                     if (isJitStressScenario(scenario)) {
2377                         if (configuration != 'Checked') {
2378                             return
2379                         }
2380                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2381                         if (isCoreFxScenario(scenario)) {
2382                             return
2383                         }
2384                     }
2385                     else if (isR2RBaselineScenario(scenario)) {
2386                         if (configuration != 'Checked' && configuration != 'Release') {
2387                             return
2388                         }
2389                     }
2390                     else if (isR2RStressScenario(scenario)) {
2391                         if (configuration != 'Checked') {
2392                             return
2393                         }
2394                     }
2395                     else {
2396                         // Skip scenarios
2397                         switch (scenario) {
2398                             case 'ilrt':
2399                             case 'longgc':
2400                             case 'gcsimulator':
2401                                 // Long GC tests take a long time on non-Release builds
2402                                 // ilrt is also Release only
2403                                 if (configuration != 'Release') {
2404                                     return
2405                                 }
2406                                 break
2407                             case 'jitdiff':
2408                                 if (configuration != 'Checked') {
2409                                     return;
2410                                 }
2411                                 break
2412                             case 'gc_reliability_framework':
2413                             case 'standalone_gc':
2414                                 if (configuration != 'Release' && configuration != 'Checked') {
2415                                     return
2416                                 }
2417                                 break
2418                             case 'formatting':
2419                                 return
2420                             case 'illink':
2421                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2422                                     return
2423                                 }
2424                                 break
2425                             case 'default':
2426                                 // Nothing skipped
2427                                 break
2428                             default:
2429                                 println("Unknown scenario: ${scenario}")
2430                                 assert false
2431                                 break
2432                         }
2433                     }
2434
2435                     // Done filtering. Now, create the jobs.
2436
2437                     def lowerConfiguration = configuration.toLowerCase()
2438                     def osGroup = getOSGroup(os)
2439                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2440
2441                     def inputCoreCLRBuildName = projectFolder + '/' +
2442                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2443
2444                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2445                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2446                     def testBuildScenario = scenario
2447                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2448                         testBuildScenario = 'default'
2449                     }
2450
2451                     def inputWindowsTestBuildArch = architecture
2452                     if (architecture == "arm64" && os != "Windows_NT") {
2453                         // Use the x64 test build for arm64 unix
2454                         inputWindowsTestBuildArch = "x64"
2455                     }
2456
2457                     def inputWindowsTestsBuildName = ""
2458
2459                     if (isJitStressScenario(scenario)) {
2460                         inputWindowsTestsBuildName = projectFolder + '/' +
2461                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2462                     } else {
2463                         inputWindowsTestsBuildName = projectFolder + '/' +
2464                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2465
2466                     }
2467
2468
2469                     // Enable Server GC for Ubuntu PR builds
2470                     def serverGCString = ''
2471                     if (os == 'Ubuntu' && isPR) {
2472                         serverGCString = '--useServerGC'
2473                     }
2474
2475                     def testOpts = ''
2476
2477                     if (isR2RScenario(scenario)) {
2478
2479                         testOpts += ' --crossgen --runcrossgentests'
2480
2481                         if (scenario == 'r2r_jitstress1') {
2482                             testOpts += ' --jitstress=1'
2483                         }
2484                         else if (scenario == 'r2r_jitstress2') {
2485                             testOpts += ' --jitstress=2'
2486                         }
2487                         else if (scenario == 'r2r_jitstressregs1') {
2488                             testOpts += ' --jitstressregs=1'
2489                         }
2490                         else if (scenario == 'r2r_jitstressregs2') {
2491                             testOpts += ' --jitstressregs=2'
2492                         }
2493                         else if (scenario == 'r2r_jitstressregs3') {
2494                             testOpts += ' --jitstressregs=3'
2495                         }
2496                         else if (scenario == 'r2r_jitstressregs4') {
2497                             testOpts += ' --jitstressregs=4'
2498                         }
2499                         else if (scenario == 'r2r_jitstressregs8') {
2500                             testOpts += ' --jitstressregs=8'
2501                         }
2502                         else if (scenario == 'r2r_jitstressregs0x10') {
2503                             testOpts += ' --jitstressregs=0x10'
2504                         }
2505                         else if (scenario == 'r2r_jitstressregs0x80') {
2506                             testOpts += ' --jitstressregs=0x80'
2507                         }
2508                         else if (scenario == 'r2r_jitstressregs0x1000') {
2509                             testOpts += ' --jitstressregs=0x1000'
2510                         }
2511                         else if (scenario == 'r2r_jitminopts') {
2512                             testOpts += ' --jitminopts'
2513                         }
2514                         else if (scenario == 'r2r_jitforcerelocs') {
2515                             testOpts += ' --jitforcerelocs'
2516                         }
2517                         else if (scenario == 'r2r_gcstress15') {
2518                             testOpts += ' --gcstresslevel=0xF'
2519                         }
2520                     }
2521                     else if (scenario == 'jitdiff') {
2522                         testOpts += ' --jitdisasm --crossgen'
2523                     }
2524                     else if (scenario == 'illink') {
2525                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2526                     }
2527                     else if (isLongGc(scenario)) {
2528                         // Long GC tests behave very poorly when they are not
2529                         // the only test running (many of them allocate until OOM).
2530                         testOpts += ' --sequential'
2531
2532                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2533                         // for running long GC and GCSimulator tests, respectively. We don't use them
2534                         // here because using a playlist file produces much more readable output on the CI machines
2535                         // and reduces running time.
2536                         //
2537                         // The Long GC playlist contains all of the tests that are
2538                         // going to be run. The GCSimulator playlist contains all of
2539                         // the GC simulator tests.
2540                         if (scenario == 'longgc') {
2541                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2542                         }
2543                         else if (scenario == 'gcsimulator') {
2544                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2545                         }
2546                     }
2547                     else if (isGcReliabilityFramework(scenario)) {
2548                         testOpts += ' --build-overlay-only'
2549                     }
2550                     else if (scenario == 'standalone_gc') {
2551                         if (osGroup == 'OSX') {
2552                             testOpts += ' --gcname=libclrgc.dylib'
2553                         }
2554                         else if (osGroup == 'Linux') {
2555                             testOpts += ' --gcname=libclrgc.so'
2556                         }
2557                         else {
2558                             println("Unexpected OS group: ${osGroup} for os ${os}")
2559                             assert false
2560                         }
2561                     }
2562
2563                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2564
2565                     def folder = getJobFolder(scenario)
2566                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2567                         // Add parameters for the inputs
2568
2569                         if (windowsArmJob == true) {
2570                             parameters {
2571                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2572                             }
2573                         }
2574                         else {
2575                             parameters {
2576                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2577                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2578                             }
2579                         }
2580
2581                         steps {
2582                             // Set up the copies
2583
2584                             // Coreclr build containing the tests and mscorlib
2585                             // pri1 jobs still need to copy windows_nt built tests
2586                             if (windowsArmJob != true) {
2587                                 copyArtifacts(inputWindowsTestsBuildName) {
2588                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2589                                     buildSelector {
2590                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2591                                     }
2592                                 }
2593                             }
2594
2595                             // Coreclr build we are trying to test
2596                             //
2597                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2598
2599                             copyArtifacts(inputCoreCLRBuildName) {
2600                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2601                                 buildSelector {
2602                                     buildNumber('${CORECLR_BUILD}')
2603                                 }
2604                             }
2605
2606                             // Windows CoreCLR Arm(64) will restore corefx
2607                             // packages correctly.
2608                             //
2609                             // In addition, test steps are entirely different
2610                             // because we do not have a unified runner
2611                             if (windowsArmJob != true) {
2612                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2613
2614                                 // HACK -- Arm64 does not have corefx jobs yet.
2615                                 // Clone corefx and build the native packages overwriting the x64 packages.
2616                                 if (architecture == 'arm64') {
2617                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2618                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2619                                 }
2620                                 else if (architecture == 'x86') {
2621                                     shell("mkdir ./bin/CoreFxNative")
2622
2623                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2624                                         includePatterns('bin/build.tar.gz')
2625                                         targetDirectory('bin/CoreFxNative')
2626                                         buildSelector {
2627                                             latestSuccessful(true)
2628                                         }
2629                                     }
2630
2631                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2632                                 }
2633
2634                                 // Unzip the tests first.  Exit with 0
2635                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2636                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2637
2638                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2639
2640                                 // Execute the tests
2641                                 def runDocker = isNeedDocker(architecture, os, false)
2642                                 def dockerPrefix = ""
2643                                 def dockerCmd = ""
2644                                 if (runDocker) {
2645                                     def dockerImage = getDockerImageName(architecture, os, false)
2646                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2647                                     dockerCmd = dockerPrefix + "${dockerImage} "
2648                                 }
2649
2650                                 // If we are running a stress mode, we'll set those variables first
2651                                 def testEnvOpt = ""
2652                                 if (isJitStressScenario(scenario)) {
2653                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2654                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2655                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2656                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2657                                     shell("${envScriptCmds}")
2658                                     testEnvOpt = "--test-env=" + scriptFileName
2659                                 }
2660
2661                                 if (isGCStressRelatedTesting(scenario)) {
2662                                     shell('./init-tools.sh')
2663                                 }
2664
2665                                 shell("""${dockerCmd}./tests/runtest.sh \\
2666                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2667                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2668                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2669                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2670
2671                                 if (isGcReliabilityFramework(scenario)) {
2672                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2673                                     if (serverGCString != '') {
2674                                         if (runDocker) {
2675                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2676                                         }
2677                                         else {
2678                                             shell("export COMPlus_gcServer=1")
2679                                         }
2680                                     }
2681
2682                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2683                                 }
2684                             }
2685
2686                             else { // windowsArmJob == true
2687                                 // Unzip tests.
2688                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2689
2690                                 // Build the build commands
2691                                 def buildCommands = ""
2692
2693                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2694                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2695                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2696
2697                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2698                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2699
2700                                 // For all jobs
2701                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2702
2703                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2704                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2705
2706                                 // ARM legacy backend; this is an altjit.
2707                                 if (architecture == "armlb") {
2708                                     addEnvVariable("COMPlus_AltJit", "*")
2709                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2710                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2711                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2712                                 }
2713
2714                                 // If we are running a stress mode, we'll set those variables as well
2715                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2716                                     def stressValues = null
2717                                     if (isJitStressScenario(scenario)) {
2718                                         stressValues = Constants.jitStressModeScenarios[scenario]
2719                                     }
2720                                     else {
2721                                         stressValues = Constants.r2rStressScenarios[scenario]
2722                                     }
2723
2724                                     stressValues.each { key, value ->
2725                                         addEnvVariable(key, value)
2726                                     }
2727                                 }
2728
2729                                 if (isR2RScenario(scenario)) {
2730                                     // Crossgen the framework assemblies.
2731                                     buildCommands += """
2732 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2733 @goto skip_PrecompileAssembly
2734
2735 :PrecompileAssembly
2736 @REM Skip mscorlib since it is already precompiled.
2737 @if /I "%3" == "mscorlib.dll" exit /b 0
2738 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2739
2740 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2741 @if "%errorlevel%" == "-2146230517" (
2742     echo %2 is not a managed assembly.
2743 ) else if "%errorlevel%" == "-2146234344" (
2744     echo %2 is not a managed assembly.
2745 ) else if %errorlevel% neq 0 (
2746     echo Unable to precompile %2
2747 ) else (
2748     echo Precompiled %2
2749 )
2750 @exit /b 0
2751
2752 :skip_PrecompileAssembly
2753 """
2754
2755                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2756                                     // crossgen on tests before running them.
2757                                     addEnvVariable("RunCrossGen", "true")
2758                                 }
2759
2760                                 // Create the smarty command
2761                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2762                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2763                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2764
2765                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2766
2767                                 if (architecture == "armlb") {
2768                                     addExclude("LEGACYJIT_FAIL")
2769                                 }
2770
2771                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2772                                     def failTag = "JITSTRESS_FAIL"
2773                                     def excludeTag = "JITSTRESS_EXCLUDE"
2774
2775                                     if (scenario.contains('gc')) {
2776                                         failTag = "GCSTRESS_FAIL"
2777                                         excludeTag = "GCSTRESS_EXCLUDE"
2778                                     }
2779
2780                                     addArchSpecificExclude(architecture, failTag)
2781                                     addArchSpecificExclude(architecture, excludeTag)
2782                                 }
2783                                 else {
2784                                     addExclude("pri1")
2785                                 }
2786
2787                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2788                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2789                                 addExclude("LONG_RUNNING")
2790
2791                                 smartyCommand += "/lstFile Tests.lst"
2792
2793                                 def testListArch = [
2794                                     'arm64': 'arm64',
2795                                     'arm': 'arm',
2796                                     'armlb': 'arm'
2797                                 ]
2798
2799                                 def archLocation = testListArch[architecture]
2800
2801                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2802                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2803                                 addCommand("${smartyCommand}")
2804
2805                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2806                                 // However, we also need to remove all the variables that were set during this batch file, so we
2807                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2808                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2809                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2810                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2811                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2812                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2813
2814                                 // ZIP up the smarty output, no matter what the smarty result.
2815                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2816
2817                                 addCommand("echo %errorlevel%")
2818                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2819
2820                                 // Use the smarty errorlevel as the script errorlevel.
2821                                 addCommand("exit /b %__save_smarty_errorlevel%")
2822
2823                                 batchFile(buildCommands)
2824                             }
2825                         }
2826                     }
2827
2828                     addToViews(newJob, isPR, architecture, os)
2829
2830                     if (scenario == 'jitdiff') {
2831                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2832                     }
2833
2834                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2835                     if (os in ['Ubuntu']) {
2836                         SummaryBuilder summaries = new SummaryBuilder()
2837                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2838                         summaries.emit(newJob)
2839                     }
2840
2841                     def affinityOptions = null
2842
2843                     if (windowsArmJob == true) {
2844                         affinityOptions = [
2845                             "use_arm64_build_machine" : false
2846                         ]
2847                     }
2848
2849                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2850                         affinityOptions = [
2851                             "large_pages" : false
2852                         ]
2853                     }
2854
2855                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2856                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2857
2858                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2859
2860                     if (windowsArmJob != true) {
2861                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2862                     }
2863                     else {
2864                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2865
2866                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2867                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2868                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2869                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2870                     }
2871
2872                     // Create a build flow to join together the build and tests required to run this test.
2873                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2874                     // Linux CoreCLR test
2875                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2876                     def fullTestJobName = projectFolder + '/' + newJob.name
2877                     // Add a reference to the input jobs for report purposes
2878                     JobReport.Report.addReference(inputCoreCLRBuildName)
2879                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2880                     JobReport.Report.addReference(fullTestJobName)
2881                     def newFlowJob = null
2882
2883                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2884                         // Do not create the flow job for RHEL jobs.
2885                         return
2886                     }
2887
2888                     // For pri0 jobs we can build tests on unix
2889                     if (windowsArmJob) {
2890                         // For Windows arm jobs there is no reason to build a parallel test job.
2891                         // The product build supports building and archiving the tests.
2892
2893                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2894                         buildFlow("""
2895 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2896
2897 // And then build the test build
2898 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2899 """)
2900                         }
2901                     }
2902                     else {
2903                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2904                         buildFlow("""
2905 // Build the input jobs in parallel
2906 parallel (
2907 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2908 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2909 )
2910
2911 // And then build the test build
2912 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2913             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2914 """)
2915                         }
2916                     }
2917
2918                     addToViews(newFlowJob, isPR, architecture, os)
2919
2920                     // For the flow jobs set the machine affinity as x64 if an armarch.
2921                     def flowArch = architecture
2922
2923                     if (flowArch in validWindowsNTCrossArches) {
2924                         flowArch = 'x64'
2925                         affinityOptions = null
2926                     }
2927
2928                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2929                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2930                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2931                 } // configuration
2932             } // os
2933         } // architecture
2934     } // isPR
2935 } // scenario
2936
2937 JobReport.Report.generateJobReport(out)
2938
2939 // Make the call to generate the help job
2940 Utilities.createHelperJob(this, project, branch,
2941     "Welcome to the ${project} Repository",  // This is prepended to the help message
2942     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2943
2944 Utilities.addCROSSCheck(this, project, branch)