netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'jitincompletehwintrinsic'       : ['COMPlus_EnableIncompleteISAClass' : '1'],
  90                'jitx86hwintrinsicnoavx'         : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
  91                'jitx86hwintrinsicnoavx2'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
  92                'jitx86hwintrinsicnosimd'        : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
  93                'jitnox86hwintrinsic'            : ['COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
  94                'corefx_baseline'                : [ : ], // corefx baseline
  95                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  96                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  97                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  98                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  99                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
 100                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
 101                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
 102                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
 103                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
 104                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 105                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 106                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 107                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 108                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 109                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 110                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 111                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 112                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 113                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 114                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 115                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 116                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 117     ]
 118
 119     // This is a set of ReadyToRun stress scenarios
 120     def static r2rStressScenarios = [
 121                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 122                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 123                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 124                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 125                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 126                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 127                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 128                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 129                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 130                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 131                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 132                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 133                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 134     ]
 135
 136     // This is the basic set of scenarios
 137     def static basicScenarios = [
 138                'default',
 139                'ilrt',
 140                'r2r',
 141                'longgc',
 142                'formatting',
 143                'gcsimulator',
 144                // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
 145                'standalone_gc',
 146                'gc_reliability_framework',
 147                'illink']
 148
 149     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 150
 151     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 152     // to Tests.lst file categories to exclude.
 153     //
 154     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 155     // and with the same values, with some commented out, for easier maintenance.
 156     //
 157     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 158     //
 159     def static validArmWindowsScenarios = [
 160                'default':                                [],
 161                // 'ilrt'
 162                'r2r':                                    ["R2R_FAIL"],
 163                // 'longgc'
 164                // 'formatting'
 165                // 'gcsimulator'
 166                // 'jitdiff'
 167                // 'standalone_gc'
 168                // 'gc_reliability_framework'
 169                // 'illink'
 170                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 184                'tieredcompilation':                      [],
 185                'forcerelocs':                            [],
 186                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 200                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 201                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 202                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 203                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 204                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 205                // 'jitsse2only'                          // Only relevant to xarch
 206                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 207                // 'corefx_baseline'
 208                // 'corefx_minopts'
 209                // 'corefx_tieredcompilation'
 210                // 'corefx_jitstress1'
 211                // 'corefx_jitstress2'
 212                // 'corefx_jitstressregs1'
 213                // 'corefx_jitstressregs2'
 214                // 'corefx_jitstressregs3'
 215                // 'corefx_jitstressregs4'
 216                // 'corefx_jitstressregs8'
 217                // 'corefx_jitstressregs0x10'
 218                // 'corefx_jitstressregs0x80'
 219                // 'corefx_jitstressregs0x1000'
 220                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 221                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 222                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 223                'heapverify1':                            [],
 224                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 225                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 226                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 227                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 230
 231                //
 232                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 233                //
 234
 235                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 236                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 249                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 250                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 251                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 252                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 253                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 254     ]
 255
 256     def static configurationList = ['Debug', 'Checked', 'Release']
 257
 258     // This is the set of architectures
 259     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 260 }
 261
 262 // **************************************************************
 263 // Create some specific views
 264 //
 265 // These aren't using the Utilities.addStandardFolderView() function, because that creates
 266 // views based on a single regular expression. These views will be generated by adding a
 267 // specific set of jobs to them.
 268 //
 269 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
 270 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
 271 // those are required, don't add them (which simplifies the view pages, as well).
 272 // **************************************************************
 273
 274 class Views {
 275     def static MergeJobView = null
 276     def static PeriodicJobView = null
 277     def static ArchitectureViews = [:]
 278     def static OSViews = [:]
 279 }
 280
 281 // MergeJobView: include all jobs that execute when a PR change is merged.
 282 Views.MergeJobView = listView('Merge') {
 283     recurse()
 284     columns {
 285         status()
 286         weather()
 287         name()
 288         lastSuccess()
 289         lastFailure()
 290         lastDuration()
 291         buildButton()
 292     }
 293 }
 294
 295 // PeriodicJobView: include all jobs that execute on a schedule
 296 Views.PeriodicJobView = listView('Periodic') {
 297     recurse()
 298     columns {
 299         status()
 300         weather()
 301         name()
 302         lastSuccess()
 303         lastFailure()
 304         lastDuration()
 305         buildButton()
 306     }
 307 }
 308
 309 // Create a view for non-PR jobs for each architecture.
 310 Constants.architectureList.each { architecture ->
 311     Views.ArchitectureViews[architecture] = listView(architecture) {
 312         recurse()
 313         columns {
 314             status()
 315             weather()
 316             name()
 317             lastSuccess()
 318             lastFailure()
 319             lastDuration()
 320             buildButton()
 321         }
 322     }
 323 }
 324
 325 // Create a view for non-PR jobs for each OS.
 326 Constants.osList.each { os ->
 327     // Don't create one for the special 'Windows_NT_BuildOnly'
 328     if (os == 'Windows_NT_BuildOnly') {
 329         return
 330     }
 331     Views.OSViews[os] = listView(os) {
 332         recurse()
 333         columns {
 334             status()
 335             weather()
 336             name()
 337             lastSuccess()
 338             lastFailure()
 339             lastDuration()
 340             buildButton()
 341         }
 342     }
 343 }
 344
 345 def static addToMergeView(def job) {
 346     Views.MergeJobView.with {
 347         jobs {
 348             name(job.name)
 349         }
 350     }
 351 }
 352
 353 def static addToPeriodicView(def job) {
 354     Views.PeriodicJobView.with {
 355         jobs {
 356             name(job.name)
 357         }
 358     }
 359 }
 360
 361 def static addToViews(def job, def isPR, def architecture, def os) {
 362     if (isPR) {
 363         // No views want PR jobs currently.
 364         return
 365     }
 366
 367     // Add to architecture view.
 368     Views.ArchitectureViews[architecture].with {
 369         jobs {
 370             name(job.name)
 371         }
 372     }
 373
 374     // Add to OS view.
 375     Views.OSViews[os].with {
 376         jobs {
 377             name(job.name)
 378         }
 379     }
 380 }
 381
 382 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
 383     addToPeriodicView(job)
 384     Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
 385 }
 386
 387 def static addGithubPushTriggerHelper(def job) {
 388     addToMergeView(job)
 389     Utilities.addGithubPushTrigger(job)
 390 }
 391
 392
 393 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 394     assert os instanceof String
 395     assert architecture instanceof String
 396
 397     def armArches = ['arm', 'armlb', 'arm64']
 398     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 399
 400     if (!(architecture in armArches)) {
 401         assert options == null
 402         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 403
 404         return
 405     }
 406
 407     // This is an arm(64) job.
 408     //
 409     // There are several options.
 410     //
 411     // Windows_NT
 412     //
 413     // Arm32 (Build) -> latest-arm64
 414     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 415     // Arm32 (Test)  -> arm64-windows_nt
 416     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 417     //
 418     // Arm64 (Build) -> latest-arm64
 419     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 420     // Arm64 (Test)  -> arm64-windows_nt
 421     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 422     //
 423     // Ubuntu
 424     //
 425     // Arm32 (Build) -> arm-cross-latest
 426     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 427     // Arm32 (Test)  -> NYI Arch not supported
 428     //       |->
 429     //
 430     // Arm64 (Build) -> arm64-cross-latest
 431     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 432     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 433     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 434     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 435     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 436
 437     // This has to be a arm arch
 438     assert architecture in armArches
 439     if (os == "Windows_NT") {
 440         // Arm(64) Windows jobs share the same machines for now
 441         def isBuild = options['use_arm64_build_machine'] == true
 442
 443         if (isBuild == true) {
 444             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 445         } else {
 446             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 447         }
 448     } else {
 449         assert os != 'Windows_NT'
 450         assert os in supportedArmLinuxOs
 451
 452         if (architecture == 'arm' || architecture == 'armlb') {
 453             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 454         } else {
 455             // Arm64 Linux
 456             if (options['is_build_only'] == true) {
 457                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 458             } else {
 459                 // Arm64 Test Machines
 460                 if (options['large_pages'] == false) {
 461                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 462                 } else {
 463                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 464                 }
 465             }
 466         }
 467     }
 468 }
 469
 470 def static isGCStressRelatedTesting(def scenario) {
 471     // The 'r2r_gcstress15' scenario is a basic scenario.
 472     // Detect it and make it a GCStress related.
 473     if (scenario == 'r2r_gcstress15')
 474     {
 475         return true;
 476     }
 477
 478     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 479     def scenarioName = scenario.toLowerCase()
 480     def isGCStressTesting = false
 481     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 482         if (k in gcStressTestEnvVars) {
 483             isGCStressTesting = true;
 484         }
 485     }
 486     return isGCStressTesting
 487 }
 488
 489 def static isCoreFxScenario(def scenario) {
 490     def corefx_prefix = 'corefx_'
 491     if (scenario.length() < corefx_prefix.length()) {
 492         return false
 493     }
 494     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 495 }
 496
 497 def static isR2RBaselineScenario(def scenario) {
 498     return (scenario == 'r2r')
 499 }
 500
 501 def static isR2RStressScenario(def scenario) {
 502     return Constants.r2rStressScenarios.containsKey(scenario)
 503 }
 504
 505 def static isR2RScenario(def scenario) {
 506     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 507 }
 508
 509 def static isJitStressScenario(def scenario) {
 510     return Constants.jitStressModeScenarios.containsKey(scenario)
 511 }
 512
 513 def static isLongGc(def scenario) {
 514     return (scenario == 'longgc' || scenario == 'gcsimulator')
 515 }
 516
 517 def static isJitDiff(def scenario) {
 518     return (scenario == 'jitdiff')
 519 }
 520
 521 def static isGcReliabilityFramework(def scenario) {
 522     return (scenario == 'gc_reliability_framework')
 523 }
 524
 525 def static isArmWindowsScenario(def scenario) {
 526     return Constants.validArmWindowsScenarios.containsKey(scenario)
 527 }
 528
 529 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 530     // 2 hours (120 minutes) is the default timeout
 531     def timeout = 120
 532
 533     if (!(scenario == 'default' && isPR == true)) {
 534         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 535         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 536         timeout = 240
 537     }
 538
 539     if (!isBuildOnly) {
 540         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 541         if (isGCStressRelatedTesting(scenario)) {
 542             timeout = 4320
 543         }
 544         else if (isCoreFxScenario(scenario)) {
 545             timeout = 360
 546         }
 547         else if (isJitStressScenario(scenario)) {
 548             timeout = 240
 549         }
 550         else if (isR2RBaselineScenario(scenario)) {
 551             timeout = 240
 552         }
 553         else if (isLongGc(scenario)) {
 554             timeout = 1440
 555         }
 556         else if (isJitDiff(scenario)) {
 557             timeout = 240
 558         }
 559         else if (isGcReliabilityFramework(scenario)) {
 560             timeout = 1440
 561         }
 562         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 563             timeout = 240
 564         }
 565     }
 566
 567     if (configuration == 'Debug') {
 568         // Debug runs can be very slow. Add an hour.
 569         timeout += 60
 570     }
 571
 572     // If we've changed the timeout from the default, set it in the job.
 573
 574     if (timeout != 120) {
 575         Utilities.setJobTimeout(newJob, timeout)
 576     }
 577 }
 578
 579 def static getJobFolder(def scenario) {
 580     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 581         return 'jitstress'
 582     }
 583     if (scenario == 'illink') {
 584         return 'illink'
 585     }
 586     return ''
 587 }
 588
 589 def static getStressModeDisplayName(def scenario) {
 590     def displayStr = ''
 591     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 592         def prefixLength = 'COMPlus_'.length()
 593         if (k.length() >= prefixLength) {
 594             def modeName = k.substring(prefixLength, k.length())
 595             displayStr += ' ' + modeName + '=' + v
 596         }
 597     }
 598
 599     if (isCoreFxScenario(scenario)) {
 600         displayStr = ('CoreFx ' + displayStr).trim()
 601     }
 602
 603     return displayStr
 604 }
 605
 606 def static getR2RDisplayName(def scenario) {
 607     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 608     def displayStr = scenario
 609     def prefixLength = 'r2r_'.length()
 610     if (displayStr.length() >= prefixLength) {
 611         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 612     } else if (scenario == 'r2r') {
 613         displayStr = "R2R"
 614     }
 615     return displayStr
 616 }
 617
 618 //
 619 // Functions to create an environment script.
 620 //      envScriptCreate -- initialize the script (call first)
 621 //      envScriptFinalize -- finalize the script (call last)
 622 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 623 //      envScriptAppendExistingScript -- append an existing script to the generated script
 624 //
 625 // Each script returns a string of commands. Concatenate all the strings together before
 626 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 627 //
 628
 629 // Initialize the environment setting script.
 630 def static envScriptCreate(def os, def stepScriptLocation) {
 631     def stepScript = ''
 632     if (os == 'Windows_NT') {
 633         stepScript += "echo Creating TestEnv script\r\n"
 634         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 635
 636         // Create at least an empty script.
 637         stepScript += "echo. > ${stepScriptLocation}\r\n"
 638     }
 639     else {
 640         stepScript += "echo Creating environment setting script\n"
 641         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 642     }
 643
 644     return stepScript
 645 }
 646
 647 // Generates the string for setting stress mode variables.
 648 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 649     def stepScript = ''
 650     if (os == 'Windows_NT') {
 651         stressModeVars.each{ k, v ->
 652             // Write out what we are writing to the script file
 653             stepScript += "echo Setting ${k}=${v}\r\n"
 654             // Write out the set itself to the script file`
 655             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 656         }
 657     }
 658     else {
 659         stressModeVars.each{ k, v ->
 660             // Write out what we are writing to the script file
 661             stepScript += "echo Setting ${k}=${v}\n"
 662             // Write out the set itself to the script file`
 663             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 664         }
 665     }
 666
 667     return stepScript
 668 }
 669
 670 // Append an existing script to an environment script.
 671 // Returns string of commands to do this.
 672 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 673     assert (os == 'Windows_NT')
 674     def stepScript = ''
 675
 676     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 677     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 678
 679     return stepScript
 680 }
 681
 682 // Finalize an environment setting script.
 683 // Returns string of commands to do this.
 684 def static envScriptFinalize(def os, def stepScriptLocation) {
 685     def stepScript = ''
 686
 687     if (os == 'Windows_NT') {
 688         // Display the resulting script. This is useful when looking at the output log file.
 689         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 690         stepScript += "type ${stepScriptLocation}\r\n"
 691     }
 692     else {
 693         stepScript += "chmod +x ${stepScriptLocation}\n"
 694     }
 695
 696     return stepScript
 697 }
 698
 699 def static isNeedDocker(def architecture, def os, def isBuild) {
 700     if (isBuild) {
 701         if (architecture == 'x86' && os == 'Ubuntu') {
 702             return true
 703         }
 704         else if (architecture == 'arm') {
 705             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 706                 return true
 707             }
 708         }
 709     }
 710     else {
 711         if (architecture == 'x86' && os == 'Ubuntu') {
 712             return true
 713         }
 714     }
 715     return false
 716 }
 717
 718 def static getDockerImageName(def architecture, def os, def isBuild) {
 719     // We must change some docker private images to official later
 720     if (isBuild) {
 721         if (architecture == 'x86' && os == 'Ubuntu') {
 722             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 723         }
 724         else if (architecture == 'arm') {
 725             if (os == 'Ubuntu') {
 726                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 727             }
 728             else if (os == 'Ubuntu16.04') {
 729                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 730             }
 731             else if (os == 'Tizen') {
 732                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 733             }
 734         }
 735     }
 736     else {
 737         if (architecture == 'x86' && os == 'Ubuntu') {
 738             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 739         }
 740     }
 741     println("Unknown architecture to use docker: ${architecture} ${os}");
 742     assert false
 743 }
 744
 745 // Calculates the name of the build job based on some typical parameters.
 746 //
 747 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 748     // If the architecture is x64, do not add that info into the build name.
 749     // Need to change around some systems and other builds to pick up the right builds
 750     // to do that.
 751
 752     def suffix = scenario != 'default' ? "_${scenario}" : '';
 753     if (isBuildOnly) {
 754         suffix += '_bld'
 755     }
 756     def baseName = ''
 757     switch (architecture) {
 758         case 'x64':
 759             if (scenario == 'default') {
 760                 // For now we leave x64 off of the name for compatibility with other jobs
 761                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 762             }
 763             else if (scenario == 'formatting') {
 764                 // we don't care about the configuration for the formatting job. It runs all configs
 765                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 766             }
 767             else {
 768                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 769             }
 770             break
 771         case 'arm64':
 772             if (os.toLowerCase() == "windows_nt") {
 773                 // These are cross builds
 774                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 775             }
 776             else {
 777                 // Defaults to a small page size set of machines.
 778                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 779             }
 780             break
 781         case 'arm':
 782             // These are cross builds
 783             if (os == 'Tizen') {
 784                 // ABI: softfp
 785                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 786             }
 787             else {
 788                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 789             }
 790             break
 791         case 'armlb':
 792             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 793             break
 794         case 'x86':
 795         case 'x86_arm_altjit':
 796         case 'x64_arm64_altjit':
 797             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 798             break
 799         default:
 800             println("Unknown architecture: ${architecture}");
 801             assert false
 802             break
 803     }
 804
 805     return baseName + suffix
 806 }
 807
 808 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 809
 810     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 811     if (os == 'Windows_NT') {
 812         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 813             if (!(branch in Constants.WindowsArm64Branches)) {
 814                 return
 815             }
 816         }
 817     }
 818
 819     // Check scenario.
 820     switch (scenario) {
 821         case 'default':
 822             switch (architecture) {
 823                 case 'x64':
 824                 case 'x86':
 825                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 826                         addPeriodicTriggerHelper(job, '@daily')
 827                     }
 828                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 829                         addGithubPushTriggerHelper(job)
 830                     }
 831                     break
 832                 case 'arm':
 833                 case 'armlb':
 834                 case 'x86_arm_altjit':
 835                 case 'x64_arm64_altjit':
 836                     addGithubPushTriggerHelper(job)
 837                     break
 838                 case 'arm64':
 839                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 840                     addPeriodicTriggerHelper(job, "H H/4 * * *")
 841                     break
 842                 default:
 843                     println("Unknown architecture: ${architecture}");
 844                     assert false
 845                     break
 846             }
 847             break
 848         case 'r2r':
 849             assert !(os in bidailyCrossList)
 850             // r2r gets a push trigger for checked/release
 851             if (configuration == 'Checked' || configuration == 'Release') {
 852                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 853                 if (architecture == 'x64' && os != 'OSX10.12') {
 854                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 855                     if (isFlowJob || os == 'Windows_NT') {
 856                         addGithubPushTriggerHelper(job)
 857                     }
 858                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 859                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 860                     if (isFlowJob) {
 861                         addPeriodicTriggerHelper(job, 'H H/12 * * *')
 862                     }
 863                 }
 864                 // For x86, only add per-commit jobs for Windows
 865                 else if (architecture == 'x86') {
 866                     if (os == 'Windows_NT') {
 867                         addGithubPushTriggerHelper(job)
 868                     }
 869                 }
 870                 // arm64 r2r jobs should only run daily.
 871                 else if (architecture == 'arm64') {
 872                     if (os == 'Windows_NT') {
 873                         addPeriodicTriggerHelper(job, '@daily')
 874                     }
 875                 }
 876             }
 877             break
 878         case 'r2r_jitstress1':
 879         case 'r2r_jitstress2':
 880         case 'r2r_jitstressregs1':
 881         case 'r2r_jitstressregs2':
 882         case 'r2r_jitstressregs3':
 883         case 'r2r_jitstressregs4':
 884         case 'r2r_jitstressregs8':
 885         case 'r2r_jitstressregs0x10':
 886         case 'r2r_jitstressregs0x80':
 887         case 'r2r_jitstressregs0x1000':
 888         case 'r2r_jitminopts':
 889         case 'r2r_jitforcerelocs':
 890         case 'r2r_gcstress15':
 891             assert !(os in bidailyCrossList)
 892
 893             // GCStress=C is currently not supported on OS X
 894             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 895                 break
 896             }
 897
 898             // GC Stress 15 r2r gets a push trigger for checked/release
 899             if (configuration == 'Checked' || configuration == 'Release') {
 900                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 901                 if (architecture == 'x64') {
 902                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 903                     if (isFlowJob || os == 'Windows_NT') {
 904                         // Add a weekly periodic trigger
 905                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 906                     }
 907                 }
 908                 // For x86, only add per-commit jobs for Windows
 909                 else if (architecture == 'x86') {
 910                     if (os == 'Windows_NT') {
 911                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 912                     }
 913                 }
 914             }
 915             break
 916         case 'longgc':
 917             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 918             assert configuration == 'Release'
 919             assert architecture == 'x64'
 920             addPeriodicTriggerHelper(job, '@daily')
 921             // TODO: Add once external email sending is available again
 922             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 923             break
 924         case 'gcsimulator':
 925             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 926             assert configuration == 'Release'
 927             assert architecture == 'x64'
 928             addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 929             // TODO: Add once external email sending is available again
 930             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 931             break
 932         case 'standalone_gc':
 933             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 934             assert (configuration == 'Release' || configuration == 'Checked')
 935             // TODO: Add once external email sending is available again
 936             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 937             addPeriodicTriggerHelper(job, '@daily')
 938             break
 939         case 'gc_reliability_framework':
 940             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 941             assert (configuration == 'Release' || configuration == 'Checked')
 942             // Only triggered by phrase.
 943             break
 944         case 'ilrt':
 945             assert !(os in bidailyCrossList)
 946             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 947             if (architecture == 'x64' && configuration == 'Release') {
 948                 // We don't expect to see a job generated except in these scenarios
 949                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 950                 if (isFlowJob || os == 'Windows_NT') {
 951                     addPeriodicTriggerHelper(job, '@daily')
 952                 }
 953             }
 954             break
 955         case 'jitdiff':
 956             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 957             assert configuration == 'Checked'
 958             assert (architecture == 'x64' || architecture == 'x86')
 959             addGithubPushTriggerHelper(job)
 960             break
 961         case 'formatting':
 962             assert (os == 'Windows_NT' || os == "Ubuntu")
 963             assert architecture == 'x64'
 964             addGithubPushTriggerHelper(job)
 965             break
 966         case 'jitstressregs1':
 967         case 'jitstressregs2':
 968         case 'jitstressregs3':
 969         case 'jitstressregs4':
 970         case 'jitstressregs8':
 971         case 'jitstressregs0x10':
 972         case 'jitstressregs0x80':
 973         case 'jitstressregs0x1000':
 974         case 'minopts':
 975         case 'forcerelocs':
 976         case 'jitstress1':
 977         case 'jitstress2':
 978         case 'jitstress2_jitstressregs1':
 979         case 'jitstress2_jitstressregs2':
 980         case 'jitstress2_jitstressregs3':
 981         case 'jitstress2_jitstressregs4':
 982         case 'jitstress2_jitstressregs8':
 983         case 'jitstress2_jitstressregs0x10':
 984         case 'jitstress2_jitstressregs0x80':
 985         case 'jitstress2_jitstressregs0x1000':
 986         case 'tailcallstress':
 987         case 'jitsse2only':
 988         case 'jitnosimd':
 989         case 'jitnox86hwintrinsic':
 990         case 'jitincompletehwintrinsic':
 991         case 'jitx86hwintrinsicnoavx':
 992         case 'jitx86hwintrinsicnoavx2':
 993         case 'jitx86hwintrinsicnosimd':
 994         case 'corefx_baseline':
 995         case 'corefx_minopts':
 996         case 'corefx_jitstress1':
 997         case 'corefx_jitstress2':
 998         case 'corefx_jitstressregs1':
 999         case 'corefx_jitstressregs2':
1000         case 'corefx_jitstressregs3':
1001         case 'corefx_jitstressregs4':
1002         case 'corefx_jitstressregs8':
1003         case 'corefx_jitstressregs0x10':
1004         case 'corefx_jitstressregs0x80':
1005         case 'corefx_jitstressregs0x1000':
1006         case 'zapdisable':
1007             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1008                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1009                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1010                     if (os == 'Windows_NT') {
1011                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1012                         addPeriodicTriggerHelper(job, '@weekly')
1013                     }
1014                 }
1015                 else {
1016                     addPeriodicTriggerHelper(job, '@daily')
1017                 }
1018             }
1019             break
1020         case 'heapverify1':
1021         case 'gcstress0x3':
1022             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
1023                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1024                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1025                     if (os == 'Windows_NT') {
1026                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1027                         addPeriodicTriggerHelper(job, '@weekly')
1028                     }
1029                     // TODO: Add once external email sending is available again
1030                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1031                 }
1032                 else {
1033                     addPeriodicTriggerHelper(job, '@weekly')
1034                 }
1035             }
1036             break
1037         case 'gcstress0xc':
1038         case 'gcstress0xc_zapdisable':
1039         case 'gcstress0xc_zapdisable_jitstress2':
1040         case 'gcstress0xc_zapdisable_heapverify1':
1041         case 'gcstress0xc_jitstress1':
1042         case 'gcstress0xc_jitstress2':
1043         case 'gcstress0xc_minopts_heapverify1':
1044             // GCStress=C is currently not supported on OS X
1045             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
1046                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1047                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
1048                     if (os == 'Windows_NT') {
1049                         // We don't have enough ARM64 machines to run these more frequently than weekly.
1050                         addPeriodicTriggerHelper(job, '@weekly')
1051                     }
1052                     // TODO: Add once external email sending is available again
1053                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
1054                 }
1055                 else {
1056                     addPeriodicTriggerHelper(job, '@weekly')
1057                 }
1058             }
1059             break
1060
1061         case 'illink':
1062             // Testing on other operating systems TBD
1063             assert (os == 'Windows_NT' || os == 'Ubuntu')
1064             if (architecture == 'x64' || architecture == 'x86') {
1065                 if (configuration == 'Checked') {
1066                     addPeriodicTriggerHelper(job, '@daily')
1067                 }
1068             }
1069             break
1070
1071         case 'tieredcompilation':
1072         case 'corefx_tieredcompilation':
1073             // No periodic jobs just yet, still testing
1074             break
1075
1076         default:
1077             println("Unknown scenario: ${scenario}");
1078             assert false
1079             break
1080     }
1081     return
1082 }
1083
1084 // **************************
1085 // Define the basic inner loop builds for PR and commit.  This is basically just the set
1086 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
1087 // builds will do a couple extra steps.
1088 // **************************
1089
1090 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
1091 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
1092 // it's just used for internal builds
1093 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1094 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1095     if (isWindowsBuildOnlyJob) {
1096         return
1097     }
1098
1099     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1100     // Non pull request builds.
1101     if (!isPR) {
1102         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1103         return
1104     }
1105
1106      def arm64Users = [
1107         'AndyAyersMS',
1108         'briansull',
1109         'BruceForstall',
1110         'CarolEidt',
1111         'cmckinsey',
1112         'erozenfeld',
1113         'janvorli',
1114         'jashook',
1115         'JosephTremoulet',
1116         'pgodeq',
1117         'pgavlin',
1118         'rartemev',
1119         'russellhadley',
1120         'RussKeldorph',
1121         'sandreenko',
1122         'sdmaclea',
1123         'swaroop-sridhar',
1124         'jkotas',
1125         'markwilkie',
1126         'weshaggard'
1127     ]
1128
1129     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
1130     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1131     def osGroup = getOSGroup(os)
1132     switch (architecture) {
1133         case 'x64': // editor brace matching: {
1134             if (scenario == 'formatting') {
1135                 assert configuration == 'Checked'
1136                 if (os == 'Windows_NT' || os == 'Ubuntu') {
1137                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
1138                 }
1139                 break
1140             }
1141
1142             switch (os) {
1143                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1144                 case 'Debian8.4':
1145                 case 'RHEL7.2':
1146                     if (scenario == 'default') {
1147                         assert !isFlowJob
1148                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1149                     }
1150                     break
1151                 case 'Ubuntu16.04':
1152                     assert !isFlowJob
1153                     assert scenario == 'default'
1154                     // Distinguish with the other architectures (arm and x86)
1155                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1156                     break
1157                 case 'Fedora24':
1158                 case 'Ubuntu16.10':
1159                     assert !isFlowJob
1160                     assert scenario == 'default'
1161                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1162                     break
1163                 case 'Ubuntu':
1164                     if (scenario == 'illink') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1166                         break
1167                     }
1168                     // fall through
1169                 case 'OSX10.12':
1170                     // Triggers on the non-flow jobs aren't necessary here
1171                     // Corefx testing uses non-flow jobs.
1172                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1173                         break
1174                     }
1175                     switch (scenario) {
1176                         case 'default':
1177                             // OSX uses checked for default PR tests
1178                             if (configuration == 'Checked') {
1179                                 // Default trigger
1180                                 assert !job.name.contains("centos")
1181                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1182                             }
1183                             break
1184                         case 'jitdiff':
1185                             if (configuration == 'Checked') {
1186                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1187                             }
1188                             break
1189                         case 'ilrt':
1190                             if (configuration == 'Release') {
1191                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1192                             }
1193                             break
1194                         case 'longgc':
1195                             if (configuration == 'Release') {
1196                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1197                             }
1198                             break
1199                         case 'gcsimulator':
1200                             if (configuration == 'Release') {
1201                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1202                             }
1203                             break
1204                         case 'standalone_gc':
1205                             if (configuration == 'Release' || configuration == 'Checked') {
1206                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1207                             }
1208                             break
1209                         case 'gc_reliability_framework':
1210                             if (configuration == 'Release' || configuration == 'Checked') {
1211                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1212                             }
1213                             break
1214                         default:
1215                             if (isJitStressScenario(scenario)) {
1216                                 def displayStr = getStressModeDisplayName(scenario)
1217                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1218                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1219                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1220                             }
1221                             else if (isR2RScenario(scenario)) {
1222                                 if (configuration == 'Release' || configuration == 'Checked') {
1223                                     def displayStr = getR2RDisplayName(scenario)
1224                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1225                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1226                                 }
1227                             }
1228                             else {
1229                                 println("Unknown scenario: ${scenario}");
1230                                 assert false
1231                             }
1232                             break
1233                     }
1234                     break
1235
1236                 case 'CentOS7.1':
1237                     switch (scenario) {
1238                         case 'default':
1239                             // CentOS uses checked for default PR tests while debug is build only
1240                             if (configuration == 'Debug') {
1241                                 // Default trigger
1242                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1243                             }
1244
1245                             // Make sure this is a flow job to get build and test.
1246                             if (configuration == 'Checked' && isFlowJob) {
1247                                 assert job.name.contains("flow")
1248                                 // Default trigger
1249                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1250                             }
1251                             break
1252                         default:
1253                             if (isR2RScenario(scenario)) {
1254                                 if (configuration == 'Release' || configuration == 'Checked') {
1255                                     def displayStr = getR2RDisplayName(scenario)
1256                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1257                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                                 }
1259                             }
1260                             break
1261                     }
1262                     break
1263
1264                 case 'Windows_NT':
1265                     switch (scenario) {
1266                         case 'default':
1267                             // Default trigger
1268                             if (configuration == 'Checked') {
1269                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1270                             }
1271                             break
1272                         case 'jitdiff':
1273                             if (configuration == 'Checked') {
1274                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1275                             }
1276                             break
1277                         case 'ilrt':
1278                             if (configuration == 'Release') {
1279                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1280                             }
1281                             break
1282                         case 'longgc':
1283                             if (configuration == 'Release') {
1284                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1285                             }
1286                             break
1287                         case 'gcsimulator':
1288                             if (configuration == 'Release') {
1289                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1290                             }
1291                             break
1292                         case 'standalone_gc':
1293                             if (configuration == 'Release' || configuration == 'Checked') {
1294                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1295                             }
1296                             break
1297                         case 'gc_reliability_framework':
1298                             if (configuration == 'Release' || configuration == 'Checked') {
1299                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1300                             }
1301                             break
1302                         case 'illink':
1303                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1304                             break
1305                         default:
1306                             if (isJitStressScenario(scenario)) {
1307                                 def displayStr = getStressModeDisplayName(scenario)
1308                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1309                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1310                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1311                             }
1312                             else if (isR2RScenario(scenario)) {
1313                                 if (configuration == 'Release' || configuration == 'Checked') {
1314                                     def displayStr = getR2RDisplayName(scenario)
1315                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1316                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1317                                 }
1318                             }
1319                             else {
1320                                 println("Unknown scenario: ${scenario}");
1321                                 assert false
1322                             }
1323                             break
1324                     }
1325                     break
1326                 default:
1327                     println("Unknown os: ${os}");
1328                     assert false
1329                     break
1330             }
1331             break
1332         // editor brace matching: }
1333         case 'armlb':
1334         case 'arm': // editor brace matching: {
1335             switch (os) {
1336                 case 'Ubuntu':
1337                 case 'Ubuntu16.04':
1338                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1339                         break
1340                     }
1341                     assert scenario == 'default'
1342                     job.with {
1343                         publishers {
1344                             azureVMAgentPostBuildAction {
1345                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1346                             }
1347                         }
1348                     }
1349                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1350                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1351                     break
1352                 case 'Tizen':
1353                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1354                         break
1355                     }
1356                     architecture='armel'
1357                     job.with {
1358                         publishers {
1359                             azureVMAgentPostBuildAction {
1360                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1361                             }
1362                         }
1363                     }
1364                     if (configuration == 'Checked') {
1365                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1366                     }
1367                     else {
1368                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1369                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1370                     }
1371                     break
1372                 case 'Windows_NT':
1373                     // Triggers on the non-flow jobs aren't necessary here
1374                     if (!isFlowJob) {
1375                         break
1376                     }
1377
1378                     // Set up a private trigger
1379                     def contextString = "${os} ${architecture} Cross ${configuration}"
1380                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1381                     if (scenario == 'default') {
1382                         contextString += " Innerloop"
1383                         triggerString += "\\W+Innerloop"
1384                     }
1385                     else {
1386                         contextString += " ${scenario}"
1387                         triggerString += "\\W+${scenario}"
1388                     }
1389
1390                     if (configuration == 'Debug') {
1391                         contextString += " Build"
1392                         triggerString += "\\W+Build"
1393                     } else {
1394                         contextString += " Build and Test"
1395                         triggerString += "\\W+Build and Test"
1396                     }
1397
1398                     triggerString += ".*"
1399
1400                     switch (scenario) {
1401                         case 'default':
1402                             // Only Checked is a default trigger.
1403                             if (configuration == 'Checked')
1404                             {
1405                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1406                             }
1407                             else
1408                             {
1409                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1410                             }
1411                             break
1412                         default:
1413                             // Stress jobs will use this code path.
1414                             if (isArmWindowsScenario(scenario)) {
1415                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1416                             }
1417                             break
1418                     }
1419                     break
1420                 default:
1421                     println("NYI os: ${os}");
1422                     assert false
1423                     break
1424             }
1425             break
1426         // editor brace matching: }
1427         case 'arm64': // editor brace matching: {
1428             // Set up a private trigger
1429             def contextString = "${os} ${architecture} Cross ${configuration}"
1430             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1431             if (scenario == 'default') {
1432                 contextString += " Innerloop"
1433                 triggerString += "\\W+Innerloop"
1434             }
1435             else {
1436                 contextString += " ${scenario}"
1437                 triggerString += "\\W+${scenario}"
1438             }
1439
1440             if (configuration == 'Debug') {
1441                 contextString += " Build"
1442                 triggerString += "\\W+Build"
1443             } else {
1444                 contextString += " Build and Test"
1445                 triggerString += "\\W+Build and Test"
1446             }
1447
1448             triggerString += ".*"
1449
1450             switch (os) {
1451                 case 'Ubuntu':
1452                 case 'Ubuntu16.04':
1453                     switch (scenario) {
1454                         case 'default':
1455                             if (configuration == 'Debug' && !isFlowJob) {
1456                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1457                             }
1458                             else {
1459                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1460                             }
1461
1462                             break
1463                         default:
1464                             if (isR2RScenario(scenario)) {
1465                                 if (configuration == 'Checked' || configuration == 'Release') {
1466                                     def displayStr = getR2RDisplayName(scenario)
1467                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1468                                 }
1469                             }
1470                             break
1471                     }
1472                     break
1473                 case 'Windows_NT':
1474                     // Triggers on the non-flow jobs aren't necessary here
1475                     if (!isFlowJob) {
1476                         break
1477                     }
1478
1479                     assert isArmWindowsScenario(scenario)
1480                     switch (scenario) {
1481                         case 'default':
1482                             if (configuration == 'Checked') {
1483                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1484                             }
1485                             else {
1486                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1487                             }
1488                             break
1489                         default:
1490                             // Stress jobs will use this code path.
1491                             if (isArmWindowsScenario(scenario)) {
1492                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1493                             }
1494                             break
1495                     }
1496                     break
1497                 default:
1498                     println("NYI os: ${os}");
1499                     assert false
1500                     break
1501             }
1502             break
1503         // editor brace matching: }
1504         case 'x86': // editor brace matching: {
1505             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1506             if (os == 'Ubuntu') {
1507                 // Triggers on the non-flow jobs aren't necessary here
1508                 if (!isFlowJob) {
1509                     break
1510                 }
1511                 // on-demand only for ubuntu x86
1512                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1513                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1514                 break
1515             }
1516             switch (scenario) {
1517                 case 'default':
1518                     if (configuration == 'Checked') {
1519                         assert !job.name.contains("centos")
1520                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1521                     }
1522                     else {
1523                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1524                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1525                     }
1526                     break
1527                 case 'ilrt':
1528                     if (configuration == 'Release') {
1529                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1530                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1531                     }
1532                     break
1533                 case 'longgc':
1534                     if (configuration == 'Release') {
1535                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1536                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1537                     }
1538                     break
1539                 case 'gcsimulator':
1540                     if (configuration == 'Release') {
1541                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1542                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1543                     }
1544                     break
1545                 case 'standalone_gc':
1546                     if (configuration == 'Release' || configuration == 'Checked') {
1547                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1548                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1549                     }
1550                     break
1551                 case 'illink':
1552                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1553                     break
1554                 default:
1555                     if (isJitStressScenario(scenario)) {
1556                         def displayStr = getStressModeDisplayName(scenario)
1557                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1558                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1559                     }
1560                     else if (isR2RScenario(scenario)) {
1561                         if (configuration == 'Release' || configuration == 'Checked') {
1562                             def displayStr = getR2RDisplayName(scenario)
1563                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1564                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1565                         }
1566                     }
1567                     else {
1568                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1569                         assert false
1570                     }
1571                     break
1572             }
1573             break
1574          // editor brace matching: }
1575         case 'x64_arm64_altjit':
1576         case 'x86_arm_altjit': // editor brace matching: {
1577             assert (os == 'Windows_NT')
1578             switch (scenario) {
1579                 case 'default':
1580                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1581                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1582                     break
1583                 default:
1584                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1585                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1586                     break
1587             }
1588             break
1589         // editor brace matching: }
1590         default:
1591             println("Unknown architecture: ${architecture}");
1592             assert false
1593             break
1594     }
1595 }
1596
1597 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1598     def buildCommands = [];
1599     def osGroup = getOSGroup(os)
1600     def lowerConfiguration = configuration.toLowerCase()
1601
1602     def priority = '1'
1603     if (scenario == 'default' && isPR == true) {
1604         priority = '0'
1605     }
1606
1607     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1608
1609     def enableCorefxTesting = isCoreFxScenario(scenario)
1610
1611     // Calculate the build steps, archival, and xunit results
1612     switch (os) {
1613         case 'Windows_NT': // editor brace matching: {
1614             switch (architecture) {
1615                 case 'x64':
1616                 case 'x86':
1617                 case 'x86_arm_altjit':
1618                 case 'x64_arm64_altjit':
1619                     def arch = architecture
1620                     def buildOpts = ''
1621                     if (architecture == 'x86_arm_altjit') {
1622                         arch = 'x86'
1623                     }
1624                     else if (architecture == 'x64_arm64_altjit') {
1625                         arch = 'x64'
1626                     }
1627
1628                     if (scenario == 'formatting') {
1629                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1630                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1631                         break
1632                     }
1633
1634                     if (scenario == 'illink') {
1635                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1636                     }
1637
1638                     // If it is a release build for windows, ensure PGO is used, else fail the build
1639                     if ((lowerConfiguration == 'release') &&
1640                         (scenario in Constants.basicScenarios) &&
1641                         (architecture != 'x86_arm_altjit') &&
1642                         (architecture != 'x64_arm64_altjit')) {
1643
1644                         buildOpts += ' -enforcepgo'
1645                     }
1646
1647                     if (enableCorefxTesting) {
1648                         buildOpts += ' skiptests';
1649                     } else {
1650                         buildOpts += " -priority=${priority}"
1651                     }
1652
1653                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1654                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1655                     // 35 characters long.
1656
1657                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1658
1659                     if (!isBuildOnly) {
1660                         def runtestArguments = ''
1661                         def testOpts = 'collectdumps'
1662
1663                         if (isR2RScenario(scenario)) {
1664
1665                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1666                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1667                             // to cause the tests to be crossgen'ed.
1668
1669                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1670                                 testOpts += ' crossgenaltjit protononjit.dll'
1671                             } else {
1672                                 testOpts += ' crossgen'
1673                             }
1674
1675                             testOpts += ' runcrossgentests'
1676                         }
1677                         else if (scenario == 'jitdiff') {
1678                             testOpts += ' jitdisasm crossgen'
1679                         }
1680                         else if (scenario == 'ilrt') {
1681                             testOpts += ' ilasmroundtrip'
1682                         }
1683                         else if (isLongGc(scenario)) {
1684                             testOpts += " ${scenario} sequential"
1685                         }
1686                         else if (scenario == 'standalone_gc') {
1687                             testOpts += ' gcname clrgc.dll'
1688                         }
1689                         else if (scenario == 'illink') {
1690                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1691                         }
1692
1693                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1694                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1695                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1696                         // which is read by the xunit harness.)
1697                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1698                         {
1699                             def timeout = 1800000
1700                             testOpts += " timeout ${timeout}"
1701                         }
1702
1703                         // If we are running a stress mode, we should write out the set of key
1704                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1705
1706                         def envScriptPath = ''
1707                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1708                             def buildCommandsStr = ''
1709                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1710                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1711
1712                             if (isJitStressScenario(scenario)) {
1713                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1714                             }
1715                             else if (isR2RStressScenario(scenario)) {
1716                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1717                             }
1718
1719                             if (architecture == 'x86_arm_altjit') {
1720                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1721                             }
1722                             else if (architecture == 'x64_arm64_altjit') {
1723                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1724                             }
1725
1726                             envScriptFinalize(os, envScriptPath)
1727
1728                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1729                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1730                             // into a single script.
1731                             buildCommands += buildCommandsStr
1732                         }
1733                         else if (architecture == 'x86_arm_altjit') {
1734                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1735                         }
1736                         else if (architecture == 'x64_arm64_altjit') {
1737                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1738                         }
1739                         if (envScriptPath != '') {
1740                             testOpts += " TestEnv ${envScriptPath}"
1741                         }
1742
1743                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1744
1745                         if (enableCorefxTesting) {
1746                             def workspaceRelativeFxRoot = "_/fx"
1747                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1748
1749                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1750
1751                             // Archive and process (only) the test results
1752                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1753                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1754
1755                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1756                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1757                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1758                         }
1759                         else if (isGcReliabilityFramework(scenario)) {
1760                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1761                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1762                         }
1763                         else {
1764                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1765                         }
1766                     }
1767
1768                     if (!enableCorefxTesting) {
1769                         // Run the rest of the build
1770                         // Build the mscorlib for the other OS's
1771                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1772                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1773
1774                         if (arch == "x64") {
1775                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1776                         }
1777
1778                         // Zip up the tests directory so that we don't use so much space/time copying
1779                         // 10s of thousands of files around.
1780                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1781
1782                         if (!isJitStressScenario(scenario)) {
1783                             // For windows, pull full test results and test drops for x86/x64.
1784                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1785                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1786                         }
1787
1788                         if (scenario == 'jitdiff') {
1789                             // retrive jit-dasm output for base commit, and run jit-diff
1790                             if (!isBuildOnly) {
1791                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1792                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1793                             }
1794                         }
1795
1796                         if (!isBuildOnly) {
1797                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1798                         }
1799                     }
1800                     break
1801                 case 'armlb':
1802                 case 'arm':
1803                     assert isArmWindowsScenario(scenario)
1804
1805                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1806                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1807
1808                     def buildArchitecture = 'arm'
1809
1810                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1811                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1812                     def armCrossgenOpt = ''
1813                     if (architecture == 'armlb') {
1814                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1815                     }
1816
1817                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1818                     priority = '1'
1819
1820                     // This is now a build only job. Do not run tests. Use the flow job.
1821                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1822
1823                     // Zip up the tests directory so that we don't use so much space/time copying
1824                     // 10s of thousands of files around.
1825                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1826
1827                     // Add archival.
1828                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1829                     break
1830                 case 'arm64':
1831                     assert isArmWindowsScenario(scenario)
1832
1833                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1834                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1835
1836                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1837                     priority = '1'
1838
1839                     // This is now a build only job. Do not run tests. Use the flow job.
1840                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1841
1842                     // Zip up the tests directory so that we don't use so much space/time copying
1843                     // 10s of thousands of files around.
1844                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1845
1846                     // Add archival.
1847                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1848                     break
1849                 default:
1850                     println("Unknown architecture: ${architecture}");
1851                     assert false
1852                     break
1853             }
1854             break
1855         // editor brace matching: }
1856         case 'Ubuntu':
1857         case 'Ubuntu16.04':
1858         case 'Ubuntu16.10':
1859         case 'Debian8.4':
1860         case 'OSX10.12':
1861         case 'CentOS7.1':
1862         case 'RHEL7.2':
1863         case 'Tizen':
1864         case 'Fedora24': // editor brace matching: {
1865             switch (architecture) {
1866                 case 'x64':
1867                 case 'x86':
1868                     if (architecture == 'x86' && os == 'Ubuntu') {
1869                         // build and PAL test
1870                         def dockerImage = getDockerImageName(architecture, os, true)
1871                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1872                         dockerImage = getDockerImageName(architecture, os, false)
1873                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1874                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1875                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1876                         break
1877                     }
1878
1879                     if (scenario == 'formatting') {
1880                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1881                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1882                         break
1883                     }
1884
1885                     if (scenario == 'illink') {
1886                         assert(os == 'Ubuntu')
1887                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1888                     }
1889
1890                     if (!enableCorefxTesting) {
1891                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1892                         // only on supported OS platforms.
1893                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1894                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1895
1896                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1897                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1898
1899                         // Basic archiving of the build
1900                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1901                         // And pal tests
1902                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1903                     }
1904                     else {
1905                         // Corefx stress testing
1906                         assert os == 'Ubuntu'
1907                         assert architecture == 'x64'
1908                         assert lowerConfiguration == 'checked'
1909                         assert isJitStressScenario(scenario)
1910
1911                         // Build coreclr
1912                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1913
1914                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1915
1916                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1917                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1918                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1919                         buildCommands += envScriptCmds
1920
1921                         // Build and text corefx
1922                         def workspaceRelativeFxRoot = "_/fx"
1923                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1924
1925                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1926
1927                         // Archive and process (only) the test results
1928                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1929                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1930                     }
1931                     break
1932                 case 'arm64':
1933                     if (!enableCorefxTesting) {
1934                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1935
1936                         // HACK -- Arm64 does not have corefx jobs yet.
1937                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1938                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1939                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1940                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1941
1942                         // Basic archiving of the build
1943                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1944                     }
1945                     break
1946                 case 'arm':
1947                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1948                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1949
1950                     // default values for Ubuntu
1951                     def arm_abi="arm"
1952                     def linuxCodeName="trusty"
1953                     if (os == 'Ubuntu16.04') {
1954                         linuxCodeName="xenial"
1955                     }
1956                     else if (os == 'Tizen') {
1957                         arm_abi="armel"
1958                         linuxCodeName="tizen"
1959                     }
1960
1961                     // Unzip the Windows test binaries first. Exit with 0
1962                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1963
1964                     // Unpack the corefx binaries
1965                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1966                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1967                     if (os != 'Tizen') {
1968                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1969                     }
1970                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1971                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1972                     if (os == 'Ubuntu') {
1973                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1974                     }
1975
1976                     // Call the ARM CI script to cross build and test using docker
1977                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1978                     --mode=docker \\
1979                     --${arm_abi} \\
1980                     --linuxCodeName=${linuxCodeName} \\
1981                     --buildConfig=${lowerConfiguration} \\
1982                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1983                     --coreFxBinDir=./bin/CoreFxBinDir \\
1984                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1985
1986                     // Basic archiving of the build, no pal tests
1987                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1988                     break
1989                 default:
1990                     println("Unknown architecture: ${architecture}");
1991                     assert false
1992                     break
1993             }
1994             break
1995         // editor brace matching: }
1996         default:
1997             println("Unknown os: ${os}");
1998             assert false
1999             break
2000     } // os
2001
2002     return buildCommands
2003 }
2004
2005 Constants.allScenarios.each { scenario ->
2006     [true, false].each { isPR ->
2007         Constants.architectureList.each { architecture ->
2008             Constants.configurationList.each { configuration ->
2009                 Constants.osList.each { os ->
2010                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2011                     // and reset the os to Windows_NT
2012                     def isBuildOnly = false
2013                     if (os == 'Windows_NT_BuildOnly') {
2014                         isBuildOnly = true
2015                         os = 'Windows_NT'
2016                     }
2017
2018                     // Tizen is only supported for arm architecture
2019                     if (os == 'Tizen' && architecture != 'arm') {
2020                         return
2021                     }
2022
2023                     // Skip totally unimplemented (in CI) configurations.
2024                     switch (architecture) {
2025                         case 'arm64':
2026                             if (os == 'Ubuntu16.04') {
2027                                 os = 'Ubuntu'
2028                             }
2029
2030                             // Windows and Ubuntu only
2031                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2032                                 return
2033                             }
2034                             break
2035                         case 'arm':
2036                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2037                                 return
2038                             }
2039                             break
2040                         case 'armlb':
2041                             if (os != 'Windows_NT') {
2042                                 return
2043                             }
2044                             break
2045                         case 'x86':
2046                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2047                                 return
2048                             }
2049                             break
2050                         case 'x86_arm_altjit':
2051                         case 'x64_arm64_altjit':
2052                             if (os != 'Windows_NT') {
2053                                 return
2054                             }
2055                             break
2056                         case 'x64':
2057                             // Everything implemented
2058                             break
2059                         default:
2060                             println("Unknown architecture: ${architecture}")
2061                             assert false
2062                             break
2063                     }
2064
2065                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2066                     // with checked builds)
2067                     if (isJitStressScenario(scenario)) {
2068                         if (configuration != 'Checked') {
2069                             return
2070                         }
2071
2072                         // Since these are just execution time differences,
2073                         // skip platforms that don't execute the tests here (Windows_NT only)
2074                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
2075                         if (!isEnabledOS || isBuildOnly) {
2076                             return
2077                         }
2078
2079                         switch (architecture) {
2080                             case 'x64':
2081                             case 'x86':
2082                             case 'x86_arm_altjit':
2083                             case 'x64_arm64_altjit':
2084                                 // x86 ubuntu: default only
2085                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2086                                     return
2087                                 }
2088                                 // Windows: Everything implemented
2089                                 break
2090
2091                             default:
2092                                 // arm, arm64, armlb: stress is handled through flow jobs.
2093                                 return
2094                         }
2095                     }
2096                     else if (isR2RScenario(scenario)) {
2097                         if (os != 'Windows_NT') {
2098                             return
2099                         }
2100                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2101                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
2102                             return
2103                         }
2104                     }
2105                     else {
2106                         // Skip scenarios
2107                         switch (scenario) {
2108                             case 'ilrt':
2109                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2110                                 // the default scenario build
2111                                 if (os != 'Windows_NT') {
2112                                     return
2113                                 }
2114                                 // Only x64 for now
2115                                 if (architecture != 'x64') {
2116                                     return
2117                                 }
2118                                 // Release only
2119                                 if (configuration != 'Release') {
2120                                     return
2121                                 }
2122                                 break
2123                             case 'jitdiff':
2124                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2125                                     return
2126                                 }
2127                                 if (architecture != 'x64') {
2128                                     return
2129                                 }
2130                                 if (configuration != 'Checked') {
2131                                     return
2132                                 }
2133                                 break
2134                             case 'longgc':
2135                             case 'gcsimulator':
2136                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2137                                     return
2138                                 }
2139                                 if (architecture != 'x64') {
2140                                     return
2141                                 }
2142                                 if (configuration != 'Release') {
2143                                     return
2144                                 }
2145                                 break
2146                             case 'gc_reliability_framework':
2147                             case 'standalone_gc':
2148                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2149                                     return
2150                                 }
2151
2152                                 if (architecture != 'x64') {
2153                                     return
2154                                 }
2155
2156                                 if (configuration != 'Release' && configuration != 'Checked') {
2157                                     return
2158                                 }
2159                                 break
2160                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2161                             case 'formatting':
2162                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2163                                     return
2164                                 }
2165                                 if (architecture != 'x64') {
2166                                     return
2167                                 }
2168                                 if (configuration != 'Checked') {
2169                                     return
2170                                 }
2171                                 if (isBuildOnly) {
2172                                     return
2173                                 }
2174                                 break
2175                             case 'illink':
2176                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2177                                     return
2178                                 }
2179                                 if (architecture != 'x64' && architecture != 'x86') {
2180                                     return
2181                                 }
2182                                 if (isBuildOnly) {
2183                                     return
2184                                 }
2185                                 break
2186                             case 'default':
2187                                 // Nothing skipped
2188                                 break
2189                             default:
2190                                 println("Unknown scenario: ${scenario}")
2191                                 assert false
2192                                 break
2193                         }
2194                     }
2195
2196                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2197                     // runtime behavior, not compile-time behavior, are not interesting.
2198                     switch (architecture) {
2199                         case 'x86_arm_altjit':
2200                         case 'x64_arm64_altjit':
2201                             if (isGCStressRelatedTesting(scenario)) {
2202                                 return
2203                             }
2204                             break
2205                         default:
2206                             break
2207                     }
2208
2209                     // Calculate names
2210                     def lowerConfiguration = configuration.toLowerCase()
2211                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2212                     def folderName = getJobFolder(scenario)
2213
2214                     // Create the new job
2215                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2216                     addToViews(newJob, isPR, architecture, os)
2217
2218                     def machineAffinityOptions = null
2219
2220                     if (os != 'Windows_NT') {
2221                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2222                     }
2223                     else {
2224                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2225                     }
2226
2227                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2228
2229                     // Add all the standard options
2230                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2231                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2232
2233                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2234                     def osGroup = getOSGroup(os)
2235
2236                     newJob.with {
2237                         steps {
2238                             if (os == 'Windows_NT') {
2239                                 buildCommands.each { buildCommand ->
2240                                     batchFile(buildCommand)
2241                                 }
2242                             }
2243                             else {
2244                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2245                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2246                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2247                                     // Define the Windows Tests and Corefx build job names
2248                                     def WindowsTestsName = projectFolder + '/' +
2249                                                            Utilities.getFullJobName(project,
2250                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2251                                                                                     false)
2252                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2253                                                        Utilities.getFolderName(branch)
2254
2255                                     // Copy the Windows test binaries and the Corefx build binaries
2256                                     copyArtifacts(WindowsTestsName) {
2257                                         includePatterns('bin/tests/tests.zip')
2258                                         buildSelector {
2259                                             latestSuccessful(true)
2260                                         }
2261                                     }
2262
2263                                     def arm_abi = 'arm'
2264                                     def corefx_os = 'linux'
2265                                     if (os == 'Tizen') {
2266                                         arm_abi = 'armel'
2267                                         corefx_os = 'tizen'
2268                                     }
2269
2270                                     // Let's use release CoreFX to test checked CoreCLR,
2271                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2272                                     def corefx_lowerConfiguration = lowerConfiguration
2273                                     if ( lowerConfiguration == 'checked' ) {
2274                                         corefx_lowerConfiguration='release'
2275                                     }
2276
2277                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2278                                         includePatterns('bin/build.tar.gz')
2279                                         buildSelector {
2280                                             latestSuccessful(true)
2281                                         }
2282                                     }
2283                                 }
2284
2285                                 buildCommands.each { buildCommand ->
2286                                     shell(buildCommand)
2287                                 }
2288                             }
2289                         }
2290                     } // newJob.with
2291
2292                 } // os
2293             } // configuration
2294         } // architecture
2295     } // isPR
2296 } // scenario
2297
2298
2299 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2300 Constants.allScenarios.each { scenario ->
2301     [true, false].each { isPR ->
2302         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2303             Constants.crossList.each { os ->
2304                 if (architecture == 'arm64') {
2305                     if (os != "Ubuntu" && os != "Windows_NT") {
2306                         return
2307                     }
2308                 } else if (architecture == 'arm' || architecture == 'armlb') {
2309                     if (os != 'Windows_NT') {
2310                         return
2311                     }
2312                 }
2313                 else if (architecture == 'x86') {
2314                     if (os != "Ubuntu") {
2315                         return
2316                     }
2317                 }
2318
2319                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2320
2321                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2322                     return
2323                 }
2324
2325                 Constants.configurationList.each { configuration ->
2326
2327                     // First, filter based on OS.
2328
2329                     if (os == 'Windows_NT') {
2330                         if (!isArmWindowsScenario(scenario)) {
2331                             return
2332                         }
2333                     }
2334                     else {
2335                         // Non-Windows
2336                         if (architecture == 'arm64') {
2337                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2338                                 return
2339                             }
2340                         }
2341                         else if (architecture == 'x86') {
2342                             // Linux/x86 only want default test
2343                             if (scenario != 'default') {
2344                                 return
2345                             }
2346                         }
2347                     }
2348
2349                     // For CentOS, we only want Checked/Release builds.
2350                     if (os == 'CentOS7.1') {
2351                         if (configuration != 'Checked' && configuration != 'Release') {
2352                             return
2353                         }
2354                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2355                             return
2356                         }
2357                     }
2358
2359                     // For RedHat and Debian, we only do Release builds.
2360                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2361                         if (configuration != 'Release') {
2362                             return
2363                         }
2364                         if (scenario != 'default') {
2365                             return
2366                         }
2367                     }
2368
2369                     // Next, filter based on scenario.
2370
2371                     if (isJitStressScenario(scenario)) {
2372                         if (configuration != 'Checked') {
2373                             return
2374                         }
2375                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2376                         if (isCoreFxScenario(scenario)) {
2377                             return
2378                         }
2379                     }
2380                     else if (isR2RBaselineScenario(scenario)) {
2381                         if (configuration != 'Checked' && configuration != 'Release') {
2382                             return
2383                         }
2384                     }
2385                     else if (isR2RStressScenario(scenario)) {
2386                         if (configuration != 'Checked') {
2387                             return
2388                         }
2389                     }
2390                     else {
2391                         // Skip scenarios
2392                         switch (scenario) {
2393                             case 'ilrt':
2394                             case 'longgc':
2395                             case 'gcsimulator':
2396                                 // Long GC tests take a long time on non-Release builds
2397                                 // ilrt is also Release only
2398                                 if (configuration != 'Release') {
2399                                     return
2400                                 }
2401                                 break
2402                             case 'jitdiff':
2403                                 if (configuration != 'Checked') {
2404                                     return;
2405                                 }
2406                                 break
2407                             case 'gc_reliability_framework':
2408                             case 'standalone_gc':
2409                                 if (configuration != 'Release' && configuration != 'Checked') {
2410                                     return
2411                                 }
2412                                 break
2413                             case 'formatting':
2414                                 return
2415                             case 'illink':
2416                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2417                                     return
2418                                 }
2419                                 break
2420                             case 'default':
2421                                 // Nothing skipped
2422                                 break
2423                             default:
2424                                 println("Unknown scenario: ${scenario}")
2425                                 assert false
2426                                 break
2427                         }
2428                     }
2429
2430                     // Done filtering. Now, create the jobs.
2431
2432                     def lowerConfiguration = configuration.toLowerCase()
2433                     def osGroup = getOSGroup(os)
2434                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2435
2436                     def inputCoreCLRBuildName = projectFolder + '/' +
2437                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2438
2439                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2440                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2441                     def testBuildScenario = scenario
2442                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2443                         testBuildScenario = 'default'
2444                     }
2445
2446                     def inputWindowsTestBuildArch = architecture
2447                     if (architecture == "arm64" && os != "Windows_NT") {
2448                         // Use the x64 test build for arm64 unix
2449                         inputWindowsTestBuildArch = "x64"
2450                     }
2451
2452                     def inputWindowsTestsBuildName = ""
2453
2454                     if (isJitStressScenario(scenario)) {
2455                         inputWindowsTestsBuildName = projectFolder + '/' +
2456                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2457                     } else {
2458                         inputWindowsTestsBuildName = projectFolder + '/' +
2459                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2460
2461                     }
2462
2463
2464                     // Enable Server GC for Ubuntu PR builds
2465                     def serverGCString = ''
2466                     if (os == 'Ubuntu' && isPR) {
2467                         serverGCString = '--useServerGC'
2468                     }
2469
2470                     def testOpts = ''
2471
2472                     if (isR2RScenario(scenario)) {
2473
2474                         testOpts += ' --crossgen --runcrossgentests'
2475
2476                         if (scenario == 'r2r_jitstress1') {
2477                             testOpts += ' --jitstress=1'
2478                         }
2479                         else if (scenario == 'r2r_jitstress2') {
2480                             testOpts += ' --jitstress=2'
2481                         }
2482                         else if (scenario == 'r2r_jitstressregs1') {
2483                             testOpts += ' --jitstressregs=1'
2484                         }
2485                         else if (scenario == 'r2r_jitstressregs2') {
2486                             testOpts += ' --jitstressregs=2'
2487                         }
2488                         else if (scenario == 'r2r_jitstressregs3') {
2489                             testOpts += ' --jitstressregs=3'
2490                         }
2491                         else if (scenario == 'r2r_jitstressregs4') {
2492                             testOpts += ' --jitstressregs=4'
2493                         }
2494                         else if (scenario == 'r2r_jitstressregs8') {
2495                             testOpts += ' --jitstressregs=8'
2496                         }
2497                         else if (scenario == 'r2r_jitstressregs0x10') {
2498                             testOpts += ' --jitstressregs=0x10'
2499                         }
2500                         else if (scenario == 'r2r_jitstressregs0x80') {
2501                             testOpts += ' --jitstressregs=0x80'
2502                         }
2503                         else if (scenario == 'r2r_jitstressregs0x1000') {
2504                             testOpts += ' --jitstressregs=0x1000'
2505                         }
2506                         else if (scenario == 'r2r_jitminopts') {
2507                             testOpts += ' --jitminopts'
2508                         }
2509                         else if (scenario == 'r2r_jitforcerelocs') {
2510                             testOpts += ' --jitforcerelocs'
2511                         }
2512                         else if (scenario == 'r2r_gcstress15') {
2513                             testOpts += ' --gcstresslevel=0xF'
2514                         }
2515                     }
2516                     else if (scenario == 'jitdiff') {
2517                         testOpts += ' --jitdisasm --crossgen'
2518                     }
2519                     else if (scenario == 'illink') {
2520                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2521                     }
2522                     else if (isLongGc(scenario)) {
2523                         // Long GC tests behave very poorly when they are not
2524                         // the only test running (many of them allocate until OOM).
2525                         testOpts += ' --sequential'
2526
2527                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2528                         // for running long GC and GCSimulator tests, respectively. We don't use them
2529                         // here because using a playlist file produces much more readable output on the CI machines
2530                         // and reduces running time.
2531                         //
2532                         // The Long GC playlist contains all of the tests that are
2533                         // going to be run. The GCSimulator playlist contains all of
2534                         // the GC simulator tests.
2535                         if (scenario == 'longgc') {
2536                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2537                         }
2538                         else if (scenario == 'gcsimulator') {
2539                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2540                         }
2541                     }
2542                     else if (isGcReliabilityFramework(scenario)) {
2543                         testOpts += ' --build-overlay-only'
2544                     }
2545                     else if (scenario == 'standalone_gc') {
2546                         if (osGroup == 'OSX') {
2547                             testOpts += ' --gcname=libclrgc.dylib'
2548                         }
2549                         else if (osGroup == 'Linux') {
2550                             testOpts += ' --gcname=libclrgc.so'
2551                         }
2552                         else {
2553                             println("Unexpected OS group: ${osGroup} for os ${os}")
2554                             assert false
2555                         }
2556                     }
2557
2558                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2559
2560                     def folder = getJobFolder(scenario)
2561                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2562                         // Add parameters for the inputs
2563
2564                         if (windowsArmJob == true) {
2565                             parameters {
2566                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2567                             }
2568                         }
2569                         else {
2570                             parameters {
2571                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2572                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2573                             }
2574                         }
2575
2576                         steps {
2577                             // Set up the copies
2578
2579                             // Coreclr build containing the tests and mscorlib
2580                             // pri1 jobs still need to copy windows_nt built tests
2581                             if (windowsArmJob != true) {
2582                                 copyArtifacts(inputWindowsTestsBuildName) {
2583                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2584                                     buildSelector {
2585                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2586                                     }
2587                                 }
2588                             }
2589
2590                             // Coreclr build we are trying to test
2591                             //
2592                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2593
2594                             copyArtifacts(inputCoreCLRBuildName) {
2595                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2596                                 buildSelector {
2597                                     buildNumber('${CORECLR_BUILD}')
2598                                 }
2599                             }
2600
2601                             // Windows CoreCLR Arm(64) will restore corefx
2602                             // packages correctly.
2603                             //
2604                             // In addition, test steps are entirely different
2605                             // because we do not have a unified runner
2606                             if (windowsArmJob != true) {
2607                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2608
2609                                 // HACK -- Arm64 does not have corefx jobs yet.
2610                                 // Clone corefx and build the native packages overwriting the x64 packages.
2611                                 if (architecture == 'arm64') {
2612                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2613                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2614                                 }
2615                                 else if (architecture == 'x86') {
2616                                     shell("mkdir ./bin/CoreFxNative")
2617
2618                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2619                                         includePatterns('bin/build.tar.gz')
2620                                         targetDirectory('bin/CoreFxNative')
2621                                         buildSelector {
2622                                             latestSuccessful(true)
2623                                         }
2624                                     }
2625
2626                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2627                                 }
2628
2629                                 // Unzip the tests first.  Exit with 0
2630                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2631                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2632
2633                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2634
2635                                 // Execute the tests
2636                                 def runDocker = isNeedDocker(architecture, os, false)
2637                                 def dockerPrefix = ""
2638                                 def dockerCmd = ""
2639                                 if (runDocker) {
2640                                     def dockerImage = getDockerImageName(architecture, os, false)
2641                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2642                                     dockerCmd = dockerPrefix + "${dockerImage} "
2643                                 }
2644
2645                                 // If we are running a stress mode, we'll set those variables first
2646                                 def testEnvOpt = ""
2647                                 if (isJitStressScenario(scenario)) {
2648                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2649                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2650                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2651                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2652                                     shell("${envScriptCmds}")
2653                                     testEnvOpt = "--test-env=" + scriptFileName
2654                                 }
2655
2656                                 if (isGCStressRelatedTesting(scenario)) {
2657                                     shell('./init-tools.sh')
2658                                 }
2659
2660                                 shell("""${dockerCmd}./tests/runtest.sh \\
2661                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2662                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2663                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2664                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2665
2666                                 if (isGcReliabilityFramework(scenario)) {
2667                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2668                                     if (serverGCString != '') {
2669                                         if (runDocker) {
2670                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2671                                         }
2672                                         else {
2673                                             shell("export COMPlus_gcServer=1")
2674                                         }
2675                                     }
2676
2677                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2678                                 }
2679                             }
2680
2681                             else { // windowsArmJob == true
2682                                 // Unzip tests.
2683                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2684
2685                                 // Build the build commands
2686                                 def buildCommands = ""
2687
2688                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2689                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2690                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2691
2692                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2693                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2694
2695                                 // For all jobs
2696                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2697
2698                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2699                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2700
2701                                 // ARM legacy backend; this is an altjit.
2702                                 if (architecture == "armlb") {
2703                                     addEnvVariable("COMPlus_AltJit", "*")
2704                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2705                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2706                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2707                                 }
2708
2709                                 // If we are running a stress mode, we'll set those variables as well
2710                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2711                                     def stressValues = null
2712                                     if (isJitStressScenario(scenario)) {
2713                                         stressValues = Constants.jitStressModeScenarios[scenario]
2714                                     }
2715                                     else {
2716                                         stressValues = Constants.r2rStressScenarios[scenario]
2717                                     }
2718
2719                                     stressValues.each { key, value ->
2720                                         addEnvVariable(key, value)
2721                                     }
2722                                 }
2723
2724                                 if (isR2RScenario(scenario)) {
2725                                     // Crossgen the framework assemblies.
2726                                     buildCommands += """
2727 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2728 @goto skip_PrecompileAssembly
2729
2730 :PrecompileAssembly
2731 @REM Skip mscorlib since it is already precompiled.
2732 @if /I "%3" == "mscorlib.dll" exit /b 0
2733 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2734
2735 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2736 @if "%errorlevel%" == "-2146230517" (
2737     echo %2 is not a managed assembly.
2738 ) else if "%errorlevel%" == "-2146234344" (
2739     echo %2 is not a managed assembly.
2740 ) else if %errorlevel% neq 0 (
2741     echo Unable to precompile %2
2742 ) else (
2743     echo Precompiled %2
2744 )
2745 @exit /b 0
2746
2747 :skip_PrecompileAssembly
2748 """
2749
2750                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2751                                     // crossgen on tests before running them.
2752                                     addEnvVariable("RunCrossGen", "true")
2753                                 }
2754
2755                                 // Create the smarty command
2756                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2757                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2758                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2759
2760                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2761
2762                                 if (architecture == "armlb") {
2763                                     addExclude("LEGACYJIT_FAIL")
2764                                 }
2765
2766                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2767                                     def failTag = "JITSTRESS_FAIL"
2768                                     def excludeTag = "JITSTRESS_EXCLUDE"
2769
2770                                     if (scenario.contains('gc')) {
2771                                         failTag = "GCSTRESS_FAIL"
2772                                         excludeTag = "GCSTRESS_EXCLUDE"
2773                                     }
2774
2775                                     addArchSpecificExclude(architecture, failTag)
2776                                     addArchSpecificExclude(architecture, excludeTag)
2777                                 }
2778                                 else {
2779                                     addExclude("pri1")
2780                                 }
2781
2782                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2783                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2784                                 addExclude("LONG_RUNNING")
2785
2786                                 smartyCommand += "/lstFile Tests.lst"
2787
2788                                 def testListArch = [
2789                                     'arm64': 'arm64',
2790                                     'arm': 'arm',
2791                                     'armlb': 'arm'
2792                                 ]
2793
2794                                 def archLocation = testListArch[architecture]
2795
2796                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2797                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2798                                 addCommand("${smartyCommand}")
2799
2800                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2801                                 // However, we also need to remove all the variables that were set during this batch file, so we
2802                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2803                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2804                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2805                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2806                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2807                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2808
2809                                 // ZIP up the smarty output, no matter what the smarty result.
2810                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2811
2812                                 addCommand("echo %errorlevel%")
2813                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2814
2815                                 // Use the smarty errorlevel as the script errorlevel.
2816                                 addCommand("exit /b %__save_smarty_errorlevel%")
2817
2818                                 batchFile(buildCommands)
2819                             }
2820                         }
2821                     }
2822
2823                     addToViews(newJob, isPR, architecture, os)
2824
2825                     if (scenario == 'jitdiff') {
2826                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2827                     }
2828
2829                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2830                     if (os in ['Ubuntu']) {
2831                         SummaryBuilder summaries = new SummaryBuilder()
2832                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2833                         summaries.emit(newJob)
2834                     }
2835
2836                     def affinityOptions = null
2837
2838                     if (windowsArmJob == true) {
2839                         affinityOptions = [
2840                             "use_arm64_build_machine" : false
2841                         ]
2842                     }
2843
2844                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2845                         affinityOptions = [
2846                             "large_pages" : false
2847                         ]
2848                     }
2849
2850                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2851                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2852
2853                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2854
2855                     if (windowsArmJob != true) {
2856                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2857                     }
2858                     else {
2859                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2860
2861                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2862                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2863                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2864                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2865                     }
2866
2867                     // Create a build flow to join together the build and tests required to run this test.
2868                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2869                     // Linux CoreCLR test
2870                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2871                     def fullTestJobName = projectFolder + '/' + newJob.name
2872                     // Add a reference to the input jobs for report purposes
2873                     JobReport.Report.addReference(inputCoreCLRBuildName)
2874                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2875                     JobReport.Report.addReference(fullTestJobName)
2876                     def newFlowJob = null
2877
2878                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2879                         // Do not create the flow job for RHEL jobs.
2880                         return
2881                     }
2882
2883                     // For pri0 jobs we can build tests on unix
2884                     if (windowsArmJob) {
2885                         // For Windows arm jobs there is no reason to build a parallel test job.
2886                         // The product build supports building and archiving the tests.
2887
2888                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2889                         buildFlow("""
2890 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2891
2892 // And then build the test build
2893 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2894 """)
2895                         }
2896                     }
2897                     else {
2898                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2899                         buildFlow("""
2900 // Build the input jobs in parallel
2901 parallel (
2902 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2903 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2904 )
2905
2906 // And then build the test build
2907 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2908             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2909 """)
2910                         }
2911                     }
2912
2913                     addToViews(newFlowJob, isPR, architecture, os)
2914
2915                     // For the flow jobs set the machine affinity as x64 if an armarch.
2916                     def flowArch = architecture
2917
2918                     if (flowArch in validWindowsNTCrossArches) {
2919                         flowArch = 'x64'
2920                         affinityOptions = null
2921                     }
2922
2923                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2924                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2925                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2926                 } // configuration
2927             } // os
2928         } // architecture
2929     } // isPR
2930 } // scenario
2931
2932 JobReport.Report.generateJobReport(out)
2933
2934 // Make the call to generate the help job
2935 Utilities.createHelperJob(this, project, branch,
2936     "Welcome to the ${project} Repository",  // This is prepended to the help message
2937     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2938
2939 Utilities.addCROSSCheck(this, project, branch)