1 // Import the utility functionality.
3 import jobs.generation.*
5 // The input project name (e.g. dotnet/coreclr)
6 def project = GithubProject
7 // The input branch name (e.g. master)
8 def branch = GithubBranchName
9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
11 // Create a folder for JIT stress jobs and associated folder views
13 Utilities.addStandardFolderView(this, 'jitstress', project)
15 // Create a folder for testing via illink
17 Utilities.addStandardFolderView(this, 'illink', project)
19 def static getOSGroup(def os) {
20 def osGroupMap = ['Ubuntu':'Linux',
22 'Ubuntu16.04': 'Linux',
23 'Ubuntu16.10': 'Linux',
27 'Windows_NT':'Windows_NT',
30 def osGroup = osGroupMap.get(os, null)
31 assert osGroup != null : "Could not find os group for ${os}"
35 def static getCrossArchitecture(def os, def architecture, def scenario) {
36 switch (architecture) {
47 // We use this class (vs variables) so that the static functions can access data here.
50 // We have very limited ARM64 hardware (used for ARM/ARM64 testing). So only allow certain branches to use it.
51 def static LimitedHardwareBranches = [
54 // Innerloop build OS's
55 // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
56 // test execution in the build flow runs. It generates the exact same build
57 // as Windows_NT but without running the tests.
63 'Windows_NT_BuildOnly',
71 def static crossList = [
79 // This is a set of JIT stress modes combined with the set of variables that
80 // need to be set to actually enable that stress mode. The key of the map is the stress mode and
81 // the values are the environment variables
82 def static jitStressModeScenarios = [
83 'minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
84 'tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
85 'no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'],
86 'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
87 'forcerelocs' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
88 'jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
89 'jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
90 'jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
91 'jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
92 'jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
93 'jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
94 'jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
95 'jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
96 'jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
97 'jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
98 'jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
99 'jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
100 'jitstress2_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
101 'jitstress2_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
102 'jitstress2_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
103 'jitstress2_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
104 'jitstress2_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
105 'jitstress2_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
106 'jitstress2_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
107 'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
108 'tailcallstress' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
109 'jitsse2only' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
110 'jitnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
111 'jitincompletehwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
112 'jitx86hwintrinsicnoavx' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
113 'jitx86hwintrinsicnoavx2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
114 'jitx86hwintrinsicnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
115 'jitnox86hwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
116 'corefx_baseline' : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
117 'corefx_minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
118 'corefx_tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
119 'corefx_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
120 'corefx_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
121 'corefx_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
122 'corefx_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
123 'corefx_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
124 'corefx_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
125 'corefx_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
126 'corefx_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
127 'corefx_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
128 'corefx_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
129 'gcstress0x3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
130 'gcstress0xc' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
131 'zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
132 'heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
133 'gcstress0xc_zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
134 'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress' : '2'],
135 'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
136 'gcstress0xc_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '1'],
137 'gcstress0xc_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '2'],
138 'gcstress0xc_minopts_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
141 // This is a set of ReadyToRun stress scenarios
142 def static r2rStressScenarios = [
143 'r2r_jitstress1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
144 'r2r_jitstress2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
145 'r2r_jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
146 'r2r_jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
147 'r2r_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
148 'r2r_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
149 'r2r_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
150 'r2r_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
151 'r2r_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
152 'r2r_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
153 'r2r_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
154 'r2r_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
155 'r2r_jitminopts' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
156 'r2r_jitforcerelocs' : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
157 'r2r_gcstress15' : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
158 'r2r_no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'],
161 // This is the basic set of scenarios
162 def static basicScenarios = [
170 // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
172 'gc_reliability_framework',
175 'crossgen_comparison']
177 def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
179 // Valid PR trigger combinations.
180 def static prTriggeredValidInnerLoopCombos = [
198 'Windows_NT_BuildOnly': [
240 // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
241 // to Tests.lst file categories to exclude.
243 // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
244 // and with the same values, with some commented out, for easier maintenance.
246 // Note that some scenarios that are commented out should be enabled, but haven't yet been.
248 def static validArmWindowsScenarios = [
252 'r2r': ["R2R_FAIL", "R2R_EXCLUDE"],
258 // 'gc_reliability_framework'
260 // 'corefx_innerloop'
261 // 'crossgen_comparison'
262 'r2r_jitstress1': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
263 'r2r_jitstress2': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
264 'r2r_jitstress1_tiered': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
265 'r2r_jitstress2_tiered': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
266 'r2r_jitstressregs1': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
267 'r2r_jitstressregs2': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
268 'r2r_jitstressregs3': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
269 'r2r_jitstressregs4': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
270 'r2r_jitstressregs8': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
271 'r2r_jitstressregs0x10': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
272 'r2r_jitstressregs0x80': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
273 'r2r_jitstressregs0x1000': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
274 'r2r_jitminopts': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
275 'r2r_jitforcerelocs': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
276 'r2r_gcstress15': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
277 'r2r_no_tiered_compilation': ["R2R_FAIL", "R2R_EXCLUDE"],
278 'minopts': ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
279 'tieredcompilation': [],
280 'no_tiered_compilation': [],
281 'no_tiered_compilation_innerloop': [],
283 'jitstress1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
284 'jitstress2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
285 'jitstress1_tiered': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
286 'jitstress2_tiered': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
287 'jitstressregs1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
288 'jitstressregs2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
289 'jitstressregs3': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
290 'jitstressregs4': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
291 'jitstressregs8': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
292 'jitstressregs0x10': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
293 'jitstressregs0x80': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
294 'jitstressregs0x1000': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
295 'jitstress2_jitstressregs1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
296 'jitstress2_jitstressregs2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
297 'jitstress2_jitstressregs3': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
298 'jitstress2_jitstressregs4': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
299 'jitstress2_jitstressregs8': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
300 'jitstress2_jitstressregs0x10': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
301 'jitstress2_jitstressregs0x80': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
302 'jitstress2_jitstressregs0x1000': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
303 'tailcallstress': ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
304 // 'jitsse2only' // Only relevant to xarch
305 'jitnosimd': [], // Only interesting on platforms where SIMD support exists.
306 // 'jitincompletehwintrinsic'
307 // 'jitx86hwintrinsicnoavx'
308 // 'jitx86hwintrinsicnoavx2'
309 // 'jitx86hwintrinsicnosimd'
310 // 'jitnox86hwintrinsic'
311 'corefx_baseline': [], // corefx tests don't use smarty
312 'corefx_minopts': [], // corefx tests don't use smarty
313 'corefx_tieredcompilation': [], // corefx tests don't use smarty
314 'corefx_jitstress1': [], // corefx tests don't use smarty
315 'corefx_jitstress2': [], // corefx tests don't use smarty
316 'corefx_jitstressregs1': [], // corefx tests don't use smarty
317 'corefx_jitstressregs2': [], // corefx tests don't use smarty
318 'corefx_jitstressregs3': [], // corefx tests don't use smarty
319 'corefx_jitstressregs4': [], // corefx tests don't use smarty
320 'corefx_jitstressregs8': [], // corefx tests don't use smarty
321 'corefx_jitstressregs0x10': [], // corefx tests don't use smarty
322 'corefx_jitstressregs0x80': [], // corefx tests don't use smarty
323 'corefx_jitstressregs0x1000': [], // corefx tests don't use smarty
324 'gcstress0x3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
325 'gcstress0xc': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
326 'zapdisable': ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
328 'gcstress0xc_zapdisable': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
329 'gcstress0xc_zapdisable_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
330 'gcstress0xc_zapdisable_heapverify1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
331 'gcstress0xc_jitstress1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
332 'gcstress0xc_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
333 'gcstress0xc_minopts_heapverify1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
336 // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
339 'minopts_zapdisable': ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
340 'gcstress0x3_jitstress1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
341 'gcstress0x3_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
342 'gcstress0x3_jitstressregs1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
343 'gcstress0x3_jitstressregs2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
344 'gcstress0x3_jitstressregs3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
345 'gcstress0x3_jitstressregs4': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
346 'gcstress0x3_jitstressregs8': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
347 'gcstress0x3_jitstressregs0x10': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
348 'gcstress0x3_jitstressregs0x80': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
349 'gcstress0x3_jitstressregs0x1000': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
350 'gcstress0xc_jitstressregs1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
351 'gcstress0xc_jitstressregs2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
352 'gcstress0xc_jitstressregs3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
353 'gcstress0xc_jitstressregs4': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
354 'gcstress0xc_jitstressregs8': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
355 'gcstress0xc_jitstressregs0x10': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
356 'gcstress0xc_jitstressregs0x80': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
357 'gcstress0xc_jitstressregs0x1000': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
360 def static validLinuxArmScenarios = [
370 // 'gc_reliability_framework'
372 // 'corefx_innerloop'
373 'crossgen_comparison',
376 'r2r_jitstress1_tiered',
377 'r2r_jitstress2_tiered',
378 'r2r_jitstressregs1',
379 'r2r_jitstressregs2',
380 'r2r_jitstressregs3',
381 'r2r_jitstressregs4',
382 'r2r_jitstressregs8',
383 'r2r_jitstressregs0x10',
384 'r2r_jitstressregs0x80',
385 'r2r_jitstressregs0x1000',
387 'r2r_jitforcerelocs',
389 'r2r_no_tiered_compilation',
392 'no_tiered_compilation',
393 'no_tiered_compilation_innerloop',
406 'jitstressregs0x1000',
407 'jitstress2_jitstressregs1',
408 'jitstress2_jitstressregs2',
409 'jitstress2_jitstressregs3',
410 'jitstress2_jitstressregs4',
411 'jitstress2_jitstressregs8',
412 'jitstress2_jitstressregs0x10',
413 'jitstress2_jitstressregs0x80',
414 'jitstress2_jitstressregs0x1000',
416 // 'jitsse2only' // Only relevant to xarch
417 // 'jitnosimd' // Only interesting on platforms where SIMD support exists.
418 // 'jitincompletehwintrinsic'
419 // 'jitx86hwintrinsicnoavx'
420 // 'jitx86hwintrinsicnoavx2'
421 // 'jitx86hwintrinsicnosimd'
422 // 'jitnox86hwintrinsic'
425 'corefx_tieredcompilation',
428 'corefx_jitstressregs1',
429 'corefx_jitstressregs2',
430 'corefx_jitstressregs3',
431 'corefx_jitstressregs4',
432 'corefx_jitstressregs8',
433 'corefx_jitstressregs0x10',
434 'corefx_jitstressregs0x80',
435 'corefx_jitstressregs0x1000',
440 'gcstress0xc_zapdisable',
441 'gcstress0xc_zapdisable_jitstress2',
442 'gcstress0xc_zapdisable_heapverify1',
443 'gcstress0xc_jitstress1',
444 'gcstress0xc_jitstress2',
445 'gcstress0xc_minopts_heapverify1'
448 def static validLinuxArm64Scenarios = [
458 // 'gc_reliability_framework'
460 // 'corefx_innerloop'
461 // 'crossgen_comparison'
464 'r2r_jitstress1_tiered',
465 'r2r_jitstress2_tiered',
466 'r2r_jitstressregs1',
467 'r2r_jitstressregs2',
468 'r2r_jitstressregs3',
469 'r2r_jitstressregs4',
470 'r2r_jitstressregs8',
471 'r2r_jitstressregs0x10',
472 'r2r_jitstressregs0x80',
473 'r2r_jitstressregs0x1000',
475 'r2r_jitforcerelocs',
491 'jitstressregs0x1000',
492 'jitstress2_jitstressregs1',
493 'jitstress2_jitstressregs2',
494 'jitstress2_jitstressregs3',
495 'jitstress2_jitstressregs4',
496 'jitstress2_jitstressregs8',
497 'jitstress2_jitstressregs0x10',
498 'jitstress2_jitstressregs0x80',
499 'jitstress2_jitstressregs0x1000',
501 // 'jitsse2only' // Only relevant to xarch
502 'jitnosimd', // Only interesting on platforms where SIMD support exists.
503 // 'jitincompletehwintrinsic'
504 // 'jitx86hwintrinsicnoavx'
505 // 'jitx86hwintrinsicnoavx2'
506 // 'jitx86hwintrinsicnosimd'
507 // 'jitnox86hwintrinsic'
510 'corefx_tieredcompilation',
513 'corefx_jitstressregs1',
514 'corefx_jitstressregs2',
515 'corefx_jitstressregs3',
516 'corefx_jitstressregs4',
517 'corefx_jitstressregs8',
518 'corefx_jitstressregs0x10',
519 'corefx_jitstressregs0x80',
520 'corefx_jitstressregs0x1000',
525 'gcstress0xc_zapdisable',
526 'gcstress0xc_zapdisable_jitstress2',
527 'gcstress0xc_zapdisable_heapverify1',
528 'gcstress0xc_jitstress1',
529 'gcstress0xc_jitstress2',
530 'gcstress0xc_minopts_heapverify1'
533 def static configurationList = ['Debug', 'Checked', 'Release']
535 // This is the set of architectures
536 // Some of these are pseudo-architectures:
537 // armem -- ARM builds/runs using an emulator. Used for Tizen runs.
538 // x86_arm_altjit -- ARM runs on x86 using the ARM altjit
539 // x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit
540 def static architectureList = ['arm', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
542 // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
543 def static armWindowsCrossArchitectureList = ['arm', 'arm64']
546 // **************************************************************
547 // Create some specific views
549 // These aren't using the Utilities.addStandardFolderView() function, because that creates
550 // views based on a single regular expression. These views will be generated by adding a
551 // specific set of jobs to them.
553 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
554 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
555 // those are required, don't add them (which simplifies the view pages, as well).
556 // **************************************************************
559 def static MergeJobView = null
560 def static PeriodicJobView = null
561 def static ArchitectureViews = [:]
562 def static OSViews = [:]
565 // MergeJobView: include all jobs that execute when a PR change is merged.
566 Views.MergeJobView = listView('Merge') {
579 // PeriodicJobView: include all jobs that execute on a schedule
580 Views.PeriodicJobView = listView('Periodic') {
593 // Create a view for non-PR jobs for each architecture.
594 Constants.architectureList.each { architecture ->
595 Views.ArchitectureViews[architecture] = listView(architecture) {
609 // Create a view for non-PR jobs for each OS.
610 Constants.osList.each { os ->
611 // Don't create one for the special 'Windows_NT_BuildOnly'
612 if (os == 'Windows_NT_BuildOnly') {
615 Views.OSViews[os] = listView(os) {
629 def static addToMergeView(def job) {
630 Views.MergeJobView.with {
637 def static addToPeriodicView(def job) {
638 Views.PeriodicJobView.with {
645 def static addToViews(def job, def isPR, def architecture, def os) {
647 // No views want PR jobs currently.
651 // Add to architecture view.
652 Views.ArchitectureViews[architecture].with {
659 Views.OSViews[os].with {
666 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
667 addToPeriodicView(job)
668 Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
671 def static addGithubPushTriggerHelper(def job) {
673 Utilities.addGithubPushTrigger(job)
677 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
678 assert os instanceof String
679 assert architecture instanceof String
681 def armArches = ['arm', 'armem', 'arm64']
683 if (!(architecture in armArches)) {
684 assert options == null
685 Utilities.setMachineAffinity(job, os, 'latest-or-auto')
690 // This is an arm(64) job.
692 // There are several options.
696 // Arm32 (Build) -> latest-arm64
697 // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
698 // Arm32 (Test) -> arm64-windows_nt
699 // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
701 // Arm64 (Build) -> latest-arm64
702 // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
703 // Arm64 (Test) -> arm64-windows_nt
704 // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
708 // Arm32 emulator (Build, Test) -> arm-cross-latest
709 // |-> os == "Tizen" && (architecture == "armem")
711 // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
712 // |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
713 // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
714 // |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
715 // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
716 // |-> os == "Ubuntu" && (architecture == "arm")
718 // Arm64 (Build) -> arm64-cross-latest
719 // |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
720 // Arm64 Small Page Size (Test) -> arm64-small-page-size
721 // |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
722 // Arm64 Large Page Size (Test) -> arm64-huge-page-size
723 // |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
725 // This has to be a arm arch
726 assert architecture in armArches
727 if (os == "Windows_NT") {
728 // arm32/arm64 Windows jobs share the same machines for now
729 def isBuild = options['use_arm64_build_machine'] == true
731 if (isBuild == true) {
732 // Current set of machines with private Windows arm64 toolset:
733 // Utilities.setMachineAffinity(job, os, 'latest-arm64')
735 // New set of machines with public Windows arm64 toolset, coming from Helix:
737 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
740 Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
743 assert os != 'Windows_NT'
745 if (architecture == 'arm64') {
746 assert os == 'Ubuntu'
747 def isFlow = (options != null) && (options['is_flow_job'] == true)
748 def isBuild = (options != null) && (options['is_build_job'] == true)
749 if (isFlow || isBuild) {
750 // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
751 // very important. Therefore, use latest or auto. Flow jobs don't need to use
753 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
755 // Arm64 Linux test machines
756 if ((options != null) && (options['large_pages'] == true)) {
757 Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
759 Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
763 else if (architecture == 'armem') {
764 // arm emulator (Tizen). Build and test on same machine,
767 Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
770 // arm Ubuntu on hardware.
771 assert architecture == 'arm'
772 assert os == 'Ubuntu'
773 def isFlow = (options != null) && (options['is_flow_job'] == true)
774 def isBuild = (options != null) && (options['is_build_job'] == true)
775 if (isFlow || isBuild) {
776 // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
777 // very important. Therefore, use latest or auto. Flow jobs don't need to use
779 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
781 // arm Ubuntu test machine
782 // There is no tag (like, e.g., "arm-latest") for this, so don't call
783 // Utilities.setMachineAffinity. Just add the machine affinity
784 // manually. We specify the Helix queue name here.
786 label('ubuntu.1404.arm32.open')
793 // setJobMachineAffinity: compute the machine affinity options for a job,
794 // then set the job with those affinity options.
795 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
797 assert (isBuildJob && !isTestJob && !isFlowJob) ||
798 (!isBuildJob && isTestJob && !isFlowJob) ||
799 (!isBuildJob && !isTestJob && isFlowJob)
801 def affinityOptions = null
802 def affinityArchitecture = architecture
804 if (os == "Windows_NT") {
805 if (architecture in Constants.armWindowsCrossArchitectureList) {
807 affinityOptions = [ "use_arm64_build_machine" : true ]
808 } else if (isTestJob) {
809 affinityOptions = [ "use_arm64_build_machine" : false ]
810 } else if (isFlowJob) {
811 // For the flow jobs set the machine affinity as x64
812 affinityArchitecture = 'x64'
817 if (architecture == 'arm64') {
819 affinityOptions = ['is_build_job': true]
820 } else if (isFlowJob) {
821 affinityOptions = ['is_flow_job': true]
822 } else if (isTestJob) {
823 affinityOptions = [ "large_pages" : false ]
826 else if (architecture == 'arm') {
828 affinityOptions = ['is_build_job': true]
829 } else if (isFlowJob) {
830 affinityOptions = ['is_flow_job': true]
835 setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
838 def static isGCStressRelatedTesting(def scenario) {
839 // The 'r2r_gcstress15' scenario is a basic scenario.
840 // Detect it and make it a GCStress related.
841 if (scenario == 'r2r_gcstress15')
846 def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
847 def scenarioName = scenario.toLowerCase()
848 def isGCStressTesting = false
849 Constants.jitStressModeScenarios[scenario].each{ k, v ->
850 if (k in gcStressTestEnvVars) {
851 isGCStressTesting = true;
854 return isGCStressTesting
857 def static isCoreFxScenario(def scenario) {
858 def corefx_prefix = 'corefx_'
859 if (scenario.length() < corefx_prefix.length()) {
862 return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
865 def static isR2RBaselineScenario(def scenario) {
866 return (scenario == 'r2r')
869 def static isR2RStressScenario(def scenario) {
870 return Constants.r2rStressScenarios.containsKey(scenario)
873 def static isR2RScenario(def scenario) {
874 return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
877 def static isJitStressScenario(def scenario) {
878 return Constants.jitStressModeScenarios.containsKey(scenario)
881 def static isLongGc(def scenario) {
882 return (scenario == 'longgc' || scenario == 'gcsimulator')
885 def static isJitDiff(def scenario) {
886 return (scenario == 'jitdiff')
889 def static isGcReliabilityFramework(def scenario) {
890 return (scenario == 'gc_reliability_framework')
893 def static isArmWindowsScenario(def scenario) {
894 return Constants.validArmWindowsScenarios.containsKey(scenario)
897 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
898 if (isBuildOnly == true) {
899 os = 'Windows_NT_BuildOnly'
902 def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
903 if (validOsPrTriggerArchConfigs != null) {
904 def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
905 if (validOsPrTriggerConfigs != null) {
906 if (configuration in validOsPrTriggerConfigs) {
915 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
916 // scheduled with a default PR trigger despite the correlation being true at the moment.
917 def static isInnerloopTestScenario(def scenario) {
918 return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
921 def static isCrossGenComparisonScenario(def scenario) {
922 return (scenario == 'crossgen_comparison')
925 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
926 assert isCrossGenComparisonScenario(scenario)
927 return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
930 def static getFxBranch(def branch) {
931 def fxBranch = branch
932 // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
933 // branch even though CoreFX doesn't have such a branch.
934 if (branch == 'dev/unix_test_workflow') {
940 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
941 // 2 hours (120 minutes) is the default timeout
944 if (!isInnerloopTestScenario(scenario)) {
945 // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
950 // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
951 if (isGCStressRelatedTesting(scenario)) {
954 else if (isCoreFxScenario(scenario)) {
957 else if (isJitStressScenario(scenario)) {
960 else if (isR2RBaselineScenario(scenario)) {
963 else if (isLongGc(scenario)) {
966 else if (isJitDiff(scenario)) {
969 else if (isGcReliabilityFramework(scenario)) {
972 else if (architecture == 'armem' || architecture == 'arm64') {
976 if (architecture == 'arm') {
977 // ARM32 machines are particularly slow.
982 if (configuration == 'Debug') {
983 // Debug runs can be very slow. Add an hour.
987 if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') {
988 // AltJit runs compile all methods twice.
992 // If we've changed the timeout from the default, set it in the job.
994 if (timeout != 120) {
995 Utilities.setJobTimeout(newJob, timeout)
999 def static getJobFolder(def scenario) {
1000 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1003 if (scenario == 'illink') {
1009 def static getStressModeDisplayName(def scenario) {
1011 Constants.jitStressModeScenarios[scenario].each{ k, v ->
1012 def prefixLength = 'COMPlus_'.length()
1013 if (k.length() >= prefixLength) {
1014 def modeName = k.substring(prefixLength, k.length())
1015 displayStr += ' ' + modeName + '=' + v
1019 if (isCoreFxScenario(scenario)) {
1020 displayStr = ('CoreFx ' + displayStr).trim()
1026 def static getR2RDisplayName(def scenario) {
1027 // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
1028 def displayStr = scenario
1029 def prefixLength = 'r2r_'.length()
1030 if (displayStr.length() >= prefixLength) {
1031 displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
1032 } else if (scenario == 'r2r') {
1038 def static getScenarioDisplayString(def scenario) {
1041 return "Innerloop Build and Test"
1043 case 'no_tiered_compilation_innerloop':
1044 def displayStr = getStressModeDisplayName(scenario)
1045 return "Innerloop Build and Test (Jit - ${displayStr})"
1047 case 'corefx_innerloop':
1048 return "CoreFX Tests"
1051 return "Build and Test"
1054 return "Jit Diff Build and Test"
1057 return "IL RoundTrip Build and Test"
1060 return "Long-Running GC Build & Test"
1063 return "GC Simulator"
1065 case 'standalone_gc':
1066 return "Standalone GC"
1068 case 'gc_reliability_framework':
1069 return "GC Reliability Framework"
1075 if (isJitStressScenario(scenario)) {
1076 def displayStr = getStressModeDisplayName(scenario)
1077 return "Build and Test (Jit - ${displayStr})"
1079 else if (isR2RScenario(scenario)) {
1080 def displayStr = getR2RDisplayName(scenario)
1081 return "${displayStr} Build and Test"
1084 return "${scenario}"
1089 println("Unknown scenario: ${scenario}");
1094 // Functions to create an environment script.
1095 // envScriptCreate -- initialize the script (call first)
1096 // envScriptFinalize -- finalize the script (call last)
1097 // envScriptSetStressModeVariables -- set stress mode variables in the env script
1098 // envScriptAppendExistingScript -- append an existing script to the generated script
1100 // Each script returns a string of commands. Concatenate all the strings together before
1101 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1104 // Initialize the environment setting script.
1105 def static envScriptCreate(def os, def stepScriptLocation) {
1107 if (os == 'Windows_NT') {
1108 stepScript += "echo Creating TestEnv script\r\n"
1109 stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1111 // Create at least an empty script.
1112 stepScript += "echo. > ${stepScriptLocation}\r\n"
1115 stepScript += "echo Creating environment setting script\n"
1116 stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1122 // Generates the string for setting stress mode variables.
1123 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1125 if (os == 'Windows_NT') {
1126 stressModeVars.each{ k, v ->
1127 // Write out what we are writing to the script file
1128 stepScript += "echo Setting ${k}=${v}\r\n"
1129 // Write out the set itself to the script file`
1130 stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1134 stressModeVars.each{ k, v ->
1135 // Write out what we are writing to the script file
1136 stepScript += "echo Setting ${k}=${v}\n"
1137 // Write out the set itself to the script file`
1138 stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1145 // Append an existing script to an environment script.
1146 // Returns string of commands to do this.
1147 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1148 assert (os == 'Windows_NT')
1151 stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1152 stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1157 // Finalize an environment setting script.
1158 // Returns string of commands to do this.
1159 def static envScriptFinalize(def os, def stepScriptLocation) {
1162 if (os == 'Windows_NT') {
1163 // Display the resulting script. This is useful when looking at the output log file.
1164 stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1165 stepScript += "type ${stepScriptLocation}\r\n"
1168 stepScript += "chmod +x ${stepScriptLocation}\n"
1174 def static isNeedDocker(def architecture, def os, def isBuild) {
1176 if (architecture == 'x86' && os == 'Ubuntu') {
1179 else if (architecture == 'armem') {
1182 else if (architecture == 'arm') {
1183 if (os == 'Ubuntu') {
1187 else if (architecture == 'arm64') {
1188 if (os == 'Ubuntu') {
1194 if (architecture == 'x86' && os == 'Ubuntu') {
1201 def static getDockerImageName(def architecture, def os, def isBuild) {
1202 // We must change some docker private images to official later
1204 if (architecture == 'x86' && os == 'Ubuntu') {
1205 return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1207 else if (architecture == 'armem') {
1208 if (os == 'Tizen') {
1209 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1212 else if (architecture == 'arm') {
1213 if (os == 'Ubuntu') {
1214 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1217 else if (architecture == 'arm64') {
1218 if (os == 'Ubuntu') {
1219 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1224 if (architecture == 'x86' && os == 'Ubuntu') {
1225 return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1228 println("Unknown architecture to use docker: ${architecture} ${os}");
1233 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1234 // and only allowing using this hardware in some specific branches.
1235 def static jobRequiresLimitedHardware(def architecture, def os) {
1236 if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
1237 // These test jobs require ARM64 hardware
1240 else if ((architecture == 'arm') && (os == 'Ubuntu')) {
1241 // These test jobs require Linux/arm32 hardware
1244 else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
1245 // These test jobs require Linux/arm64 hardware
1253 // Calculates the name of the build job based on some typical parameters.
1255 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1256 // If the architecture is x64, do not add that info into the build name.
1257 // Need to change around some systems and other builds to pick up the right builds
1260 def suffix = scenario != 'normal' ? "_${scenario}" : '';
1265 switch (architecture) {
1267 if (scenario == 'normal') {
1268 // For now we leave x64 off of the name for compatibility with other jobs
1269 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1271 else if (scenario == 'formatting') {
1272 // we don't care about the configuration for the formatting job. It runs all configs
1273 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1276 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1280 if (os.toLowerCase() == "windows_nt") {
1281 // These are cross builds
1282 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1285 // Defaults to a small page size set of machines.
1286 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
1290 // These are cross builds
1291 assert os == 'Tizen'
1292 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1295 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1298 case 'x86_arm_altjit':
1299 case 'x64_arm64_altjit':
1300 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1303 println("Unknown architecture: ${architecture}");
1308 return baseName + suffix
1311 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1312 def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1314 // Limited hardware is restricted for non-PR triggers to certain branches.
1315 if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1319 // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
1320 if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1324 // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1325 // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1326 if (architecture == 'x86' && os == 'Ubuntu') {
1333 case 'no_tiered_compilation_innerloop':
1334 case 'crossgen_comparison':
1337 switch (architecture) {
1340 if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1341 addPeriodicTriggerHelper(job, '@daily')
1343 else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
1344 addGithubPushTriggerHelper(job)
1348 if (os == 'Windows_NT') {
1349 if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1350 // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1351 // Do the builds daily.
1352 addPeriodicTriggerHelper(job, '@daily')
1356 // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1358 addPeriodicTriggerHelper(job, '@daily')
1363 if (os == 'Windows_NT') {
1364 if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1365 // We would normally want a push trigger, but with limited hardware we can't keep up.
1366 // Do the builds daily.
1367 addPeriodicTriggerHelper(job, '@daily')
1371 assert os == 'Ubuntu'
1372 // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1374 // Currently no push triggers, with limited arm Linux hardware.
1375 // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1376 addPeriodicTriggerHelper(job, '@daily')
1381 addGithubPushTriggerHelper(job)
1383 case 'x86_arm_altjit':
1384 case 'x64_arm64_altjit':
1385 // Only do altjit push triggers for Checked; don't waste time on Debug or Release.
1386 if (configuration == 'Checked') {
1387 addGithubPushTriggerHelper(job)
1391 println("Unknown architecture: ${architecture}");
1397 assert !(os in bidailyCrossList)
1398 // r2r gets a push trigger for checked/release
1399 if (configuration == 'Checked' || configuration == 'Release') {
1400 assert (os == 'Windows_NT') || (os in Constants.crossList)
1401 if (architecture == 'x64' && os != 'OSX10.12') {
1402 //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1403 if (isFlowJob || os == 'Windows_NT') {
1404 addGithubPushTriggerHelper(job)
1406 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1407 } else if (architecture == 'x64' && os == 'OSX10.12'){
1409 addPeriodicTriggerHelper(job, 'H H/12 * * *')
1412 // For x86, only add per-commit jobs for Windows
1413 else if (architecture == 'x86') {
1414 if (os == 'Windows_NT') {
1415 addGithubPushTriggerHelper(job)
1418 // arm64 r2r jobs should only run weekly.
1419 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1420 else if (architecture == 'arm64') {
1421 if (os == 'Windows_NT') {
1423 addPeriodicTriggerHelper(job, '@weekly')
1427 // arm r2r jobs should only run weekly.
1428 else if (architecture == 'arm') {
1430 addPeriodicTriggerHelper(job, '@weekly')
1435 case 'r2r_jitstress1':
1436 case 'r2r_jitstress2':
1437 case 'r2r_jitstress1_tiered':
1438 case 'r2r_jitstress2_tiered':
1439 case 'r2r_jitstressregs1':
1440 case 'r2r_jitstressregs2':
1441 case 'r2r_jitstressregs3':
1442 case 'r2r_jitstressregs4':
1443 case 'r2r_jitstressregs8':
1444 case 'r2r_jitstressregs0x10':
1445 case 'r2r_jitstressregs0x80':
1446 case 'r2r_jitstressregs0x1000':
1447 case 'r2r_jitminopts':
1448 case 'r2r_jitforcerelocs':
1449 case 'r2r_gcstress15':
1450 case 'r2r_no_tiered_compilation':
1451 assert !(os in bidailyCrossList)
1453 // GCStress=C is currently not supported on OS X
1454 if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1458 // GC Stress 15 r2r gets a push trigger for checked/release
1459 if (configuration == 'Checked' || configuration == 'Release') {
1460 assert (os == 'Windows_NT') || (os in Constants.crossList)
1461 if (architecture == 'x64') {
1462 //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1463 if (isFlowJob || os == 'Windows_NT') {
1464 // Add a weekly periodic trigger
1465 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1468 // For x86, only add periodic jobs for Windows
1469 else if (architecture == 'x86') {
1470 if (os == 'Windows_NT') {
1471 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1474 // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
1475 else if (architecture == 'arm64') {
1476 if (os == 'Windows_NT') {
1478 addPeriodicTriggerHelper(job, '@weekly')
1482 else if (architecture == 'arm') {
1484 addPeriodicTriggerHelper(job, '@weekly')
1490 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1491 assert configuration == 'Release'
1492 assert architecture == 'x64'
1493 addPeriodicTriggerHelper(job, '@daily')
1494 // TODO: Add once external email sending is available again
1495 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1498 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1499 assert configuration == 'Release'
1500 assert architecture == 'x64'
1501 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1502 // TODO: Add once external email sending is available again
1503 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1505 case 'standalone_gc':
1506 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1507 assert (configuration == 'Release' || configuration == 'Checked')
1508 // TODO: Add once external email sending is available again
1509 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1510 addPeriodicTriggerHelper(job, '@daily')
1512 case 'gc_reliability_framework':
1513 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1514 assert (configuration == 'Release' || configuration == 'Checked')
1515 // Only triggered by phrase.
1518 assert !(os in bidailyCrossList)
1519 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1520 if (architecture == 'x64' && configuration == 'Release') {
1521 // We don't expect to see a job generated except in these scenarios
1522 assert (os == 'Windows_NT') || (os in Constants.crossList)
1523 if (isFlowJob || os == 'Windows_NT') {
1524 addPeriodicTriggerHelper(job, '@daily')
1529 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1530 assert configuration == 'Checked'
1531 assert (architecture == 'x64' || architecture == 'x86')
1532 addGithubPushTriggerHelper(job)
1535 assert (os == 'Windows_NT' || os == "Ubuntu")
1536 assert architecture == 'x64'
1537 addGithubPushTriggerHelper(job)
1539 case 'jitstressregs1':
1540 case 'jitstressregs2':
1541 case 'jitstressregs3':
1542 case 'jitstressregs4':
1543 case 'jitstressregs8':
1544 case 'jitstressregs0x10':
1545 case 'jitstressregs0x80':
1546 case 'jitstressregs0x1000':
1548 case 'tieredcompilation':
1549 case 'no_tiered_compilation':
1553 case 'jitstress1_tiered':
1554 case 'jitstress2_tiered':
1555 case 'jitstress2_jitstressregs1':
1556 case 'jitstress2_jitstressregs2':
1557 case 'jitstress2_jitstressregs3':
1558 case 'jitstress2_jitstressregs4':
1559 case 'jitstress2_jitstressregs8':
1560 case 'jitstress2_jitstressregs0x10':
1561 case 'jitstress2_jitstressregs0x80':
1562 case 'jitstress2_jitstressregs0x1000':
1563 case 'tailcallstress':
1566 case 'jitnox86hwintrinsic':
1567 case 'jitincompletehwintrinsic':
1568 case 'jitx86hwintrinsicnoavx':
1569 case 'jitx86hwintrinsicnoavx2':
1570 case 'jitx86hwintrinsicnosimd':
1571 case 'corefx_baseline':
1572 case 'corefx_minopts':
1573 case 'corefx_tieredcompilation':
1574 case 'corefx_jitstress1':
1575 case 'corefx_jitstress2':
1576 case 'corefx_jitstressregs1':
1577 case 'corefx_jitstressregs2':
1578 case 'corefx_jitstressregs3':
1579 case 'corefx_jitstressregs4':
1580 case 'corefx_jitstressregs8':
1581 case 'corefx_jitstressregs0x10':
1582 case 'corefx_jitstressregs0x80':
1583 case 'corefx_jitstressregs0x1000':
1585 if (os == 'CentOS7.1') {
1588 if (os in bidailyCrossList) {
1591 // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1592 // build for the flow job. We don't need cron jobs for these. Note that the
1593 // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1594 // function very early, so only non-Windows gets here.
1595 if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1598 if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1601 assert (os == 'Windows_NT') || (os in Constants.crossList)
1602 if (jobRequiresLimitedHardware(architecture, os)) {
1603 addPeriodicTriggerHelper(job, '@weekly')
1606 addPeriodicTriggerHelper(job, '@daily')
1611 if (os == 'CentOS7.1') {
1614 if (os in bidailyCrossList) {
1617 if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1618 // TODO: should we have cron jobs for arm64 Linux GCStress?
1621 assert (os == 'Windows_NT') || (os in Constants.crossList)
1622 addPeriodicTriggerHelper(job, '@weekly')
1625 case 'gcstress0xc_zapdisable':
1626 case 'gcstress0xc_zapdisable_jitstress2':
1627 case 'gcstress0xc_zapdisable_heapverify1':
1628 case 'gcstress0xc_jitstress1':
1629 case 'gcstress0xc_jitstress2':
1630 case 'gcstress0xc_minopts_heapverify1':
1631 if (os == 'CentOS7.1') {
1634 if (os == 'OSX10.12') {
1635 // GCStress=C is currently not supported on OS X
1638 if (os in bidailyCrossList) {
1641 if ((architecture == 'arm64') && (os != 'Windows_NT')) {
1642 // TODO: should we have cron jobs for arm64 Linux GCStress?
1645 assert (os == 'Windows_NT') || (os in Constants.crossList)
1646 addPeriodicTriggerHelper(job, '@weekly')
1650 // Testing on other operating systems TBD
1651 assert (os == 'Windows_NT' || os == 'Ubuntu')
1652 if (architecture == 'x64' || architecture == 'x86') {
1653 if (configuration == 'Checked') {
1654 addPeriodicTriggerHelper(job, '@daily')
1660 println("Unknown scenario: ${scenario}");
1667 // **************************
1668 // Define the basic inner loop builds for PR and commit. This is basically just the set
1669 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked. In addition, the windows
1670 // builds will do a couple extra steps.
1671 // **************************
1673 // Adds a trigger for the PR build if one is needed. If isFlowJob is true, then this is the
1674 // flow job that rolls up the build and test for non-windows OS's. // If the job is a windows build only job,
1675 // it's just used for internal builds
1676 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1677 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1678 def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1680 if (isWindowsBuildOnlyJob) {
1684 def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1685 // Non pull request builds.
1687 addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1712 // Pull request builds. Generally these fall into two categories: default triggers and on-demand triggers
1713 // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1715 def contextString = ""
1716 def triggerString = ""
1717 def needsTrigger = true
1718 def isDefaultTrigger = false
1719 def isArm64PrivateJob = false
1720 def scenarioString = ""
1722 // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1723 // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1724 // some jobs where the Debug configuration just does builds, no tests.
1726 // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1727 // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1728 // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1729 // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1731 if (architecture == 'armem') {
1732 assert os == 'Tizen'
1733 architecture = 'armel'
1736 switch (architecture) {
1737 case 'x64_arm64_altjit':
1738 case 'x86_arm_altjit':
1739 // TODO: for consistency, add "Build and Test" at end.
1740 contextString = "${os} ${architecture} ${configuration} ${scenario}"
1741 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*"
1747 contextString = "${os} ${architecture} Cross ${configuration}"
1748 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1750 if (scenario == 'innerloop') {
1751 contextString += " Innerloop"
1752 triggerString += "\\W+Innerloop"
1755 contextString += " ${scenario}"
1756 triggerString += "\\W+${scenario}"
1759 if (configuration == 'Debug') {
1760 contextString += " Build"
1761 triggerString += "\\W+Build"
1764 contextString += " Build and Test"
1765 triggerString += "\\W+Build and Test"
1768 triggerString += ".*"
1772 scenarioString = getScenarioDisplayString(scenario)
1773 contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1774 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1778 triggerString += "\\W+Build and Test.*"
1781 case 'corefx_innerloop': // maintain this asymmetry
1782 triggerString += "\\W+CoreFX Tests.*"
1786 triggerString += "\\W+${scenario}.*"
1790 triggerString += ".*"
1794 // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1795 // non-flow jobs that are only used as part of flow jobs.
1797 switch (architecture) {
1798 case 'x64': // editor brace matching: {
1799 if (scenario == 'formatting') {
1800 assert configuration == 'Checked'
1801 if (os == 'Windows_NT' || os == 'Ubuntu') {
1802 isDefaultTrigger = true
1803 contextString = "${os} ${architecture} Formatting"
1809 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1812 if (scenario == 'innerloop') {
1814 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1815 isDefaultTrigger = true
1825 assert scenario != 'innerloop'
1826 contextString = "${os} ${architecture} ${configuration} Build"
1827 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1831 if (scenario == 'illink') {
1834 else if (scenario == 'corefx_innerloop') {
1835 if (configuration == 'Checked') {
1836 isDefaultTrigger = true
1844 // Triggers on the non-flow jobs aren't necessary here
1845 // Corefx testing uses non-flow jobs.
1846 if (!isFlowJob && !isCoreFxScenario(scenario)) {
1847 needsTrigger = false
1852 isDefaultTrigger = true
1855 case 'no_tiered_compilation_innerloop':
1856 if (os == 'Ubuntu') {
1857 isDefaultTrigger = true
1869 // CentOS uses checked for default PR tests while debug is build only
1870 if (configuration == 'Debug') {
1871 isDefaultTrigger = true
1872 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1876 // Make sure this is a flow job to get build and test.
1878 needsTrigger = false
1882 if (configuration == 'Checked') {
1883 assert job.name.contains("flow")
1884 isDefaultTrigger = true
1885 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1890 // Make sure this is a flow job to get build and test.
1892 needsTrigger = false
1905 case 'no_tiered_compilation_innerloop':
1906 isDefaultTrigger = true
1909 case 'corefx_innerloop':
1910 if (configuration == 'Checked' || configuration == 'Release') {
1911 isDefaultTrigger = true
1921 println("Unknown os: ${os}");
1928 // editor brace matching: }
1930 case 'armel': // editor brace matching: {
1933 azureVMAgentPostBuildAction {
1934 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1941 if (scenario == 'innerloop') {
1942 if (configuration == 'Checked') {
1943 isDefaultTrigger = true
1950 // editor brace matching: }
1953 case 'arm64': // editor brace matching: {
1957 // TODO: make arm and arm64 Ubuntu more alike
1959 if (architecture == 'arm') {
1960 // Triggers on the non-flow jobs aren't necessary
1962 needsTrigger = false
1968 case 'no_tiered_compilation_innerloop':
1969 if (configuration == 'Checked') {
1970 isDefaultTrigger = true
1976 assert architecture == 'arm64'
1980 if (configuration == 'Debug' && !isFlowJob) {
1981 isDefaultTrigger = true
1989 assert isArmWindowsScenario(scenario)
1991 // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1992 // the non-flow Build job. All others need a trigger on the flow job.
1993 def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1994 if (isFlowJob != needsFlowJobTrigger) {
1995 needsTrigger = false
2001 if (configuration == 'Debug') {
2002 // Add default PR trigger for Windows arm64 Debug builds. This is a build only -- no tests are run --
2003 // so the private test hardware is not used. Thus, it can be run by all users, not just arm64Users.
2004 // People in arm64Users will get both this and the Checked Build and Test job.
2005 isDefaultTrigger = true
2006 } else if (configuration == 'Checked') {
2007 isDefaultTrigger = true
2008 isArm64PrivateJob = true
2012 isArm64PrivateJob = true
2017 println("NYI os: ${os}");
2023 // editor brace matching: }
2024 case 'x86': // editor brace matching: {
2025 assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
2026 if (os == 'Ubuntu') {
2027 // Triggers on the non-flow jobs aren't necessary here
2029 needsTrigger = false
2033 // on-demand only for ubuntu x86
2034 contextString = "${os} ${architecture} ${configuration} Build"
2035 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
2040 case 'no_tiered_compilation_innerloop':
2041 isDefaultTrigger = true
2048 // editor brace matching: }
2049 case 'x64_arm64_altjit':
2050 case 'x86_arm_altjit':
2051 // Everything default
2055 println("Unknown architecture: ${architecture}");
2061 if (isArm64PrivateJob) {
2062 if (isDefaultTrigger) {
2063 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2066 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2070 if (isDefaultTrigger) {
2071 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2074 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2080 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2081 def buildCommands = []
2082 def osGroup = getOSGroup(os)
2083 def lowerConfiguration = configuration.toLowerCase()
2086 if (isInnerloopTestScenario(scenario)) {
2090 def doCoreFxTesting = isCoreFxScenario(scenario)
2092 // Calculate the build steps, archival, and xunit results
2094 case 'Windows_NT': // editor brace matching: {
2095 switch (architecture) {
2098 case 'x86_arm_altjit':
2099 case 'x64_arm64_altjit':
2100 def arch = architecture
2102 if (architecture == 'x86_arm_altjit') {
2105 else if (architecture == 'x64_arm64_altjit') {
2109 if (scenario == 'formatting') {
2110 buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2111 Utilities.addArchival(newJob, "format.patch", "", true, false)
2115 if (scenario == 'illink') {
2116 buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2119 // If it is a release build for Windows, ensure PGO is used, else fail the build.
2120 if ((lowerConfiguration == 'release') &&
2121 (scenario in Constants.basicScenarios) &&
2122 (architecture != 'x86_arm_altjit') &&
2123 (architecture != 'x64_arm64_altjit')) {
2125 buildOpts += ' -enforcepgo'
2128 if (doCoreFxTesting) {
2129 buildOpts += ' skiptests';
2131 buildOpts += " -priority=${priority}"
2134 // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2135 // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2136 // 35 characters long.
2138 buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2141 def runtestArguments = ''
2142 def testOpts = 'collectdumps'
2144 if (isR2RScenario(scenario)) {
2146 // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
2147 // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2148 // to cause the tests to be crossgen'ed.
2150 if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
2151 testOpts += ' crossgenaltjit protononjit.dll'
2153 testOpts += ' crossgen'
2156 testOpts += ' runcrossgentests'
2158 else if (scenario == 'jitdiff') {
2159 testOpts += ' jitdisasm crossgen'
2161 else if (scenario == 'ilrt') {
2162 testOpts += ' ilasmroundtrip'
2164 else if (isLongGc(scenario)) {
2165 testOpts += " ${scenario} sequential"
2167 else if (scenario == 'standalone_gc') {
2168 testOpts += ' gcname clrgc.dll'
2170 else if (scenario == 'illink') {
2171 testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2174 // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2175 // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2176 // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2177 // which is read by the xunit harness.)
2178 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2180 def timeout = 1800000
2181 testOpts += " timeout ${timeout}"
2184 // If we are running a stress mode, we should write out the set of key
2185 // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2187 def envScriptPath = ''
2188 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2189 def buildCommandsStr = ''
2190 envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2191 buildCommandsStr += envScriptCreate(os, envScriptPath)
2193 if (isJitStressScenario(scenario)) {
2194 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2196 else if (isR2RStressScenario(scenario)) {
2197 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2200 if (architecture == 'x86_arm_altjit') {
2201 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
2203 else if (architecture == 'x64_arm64_altjit') {
2204 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
2207 envScriptFinalize(os, envScriptPath)
2209 // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2210 // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2211 // into a single script.
2212 buildCommands += buildCommandsStr
2214 else if (architecture == 'x86_arm_altjit') {
2215 envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
2217 else if (architecture == 'x64_arm64_altjit') {
2218 envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
2220 if (envScriptPath != '') {
2221 testOpts += " TestEnv ${envScriptPath}"
2224 runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2226 if (doCoreFxTesting) {
2227 if (scenario == 'corefx_innerloop') {
2228 // Create CORE_ROOT and testhost
2229 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2230 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2232 // Archive and process (only) the test results
2233 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2234 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2237 def workspaceRelativeFxRoot = "_/fx"
2238 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2239 def fxBranch = getFxBranch(branch)
2241 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
2243 // Archive and process (only) the test results
2244 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2245 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2247 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2248 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2249 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
2252 else if (isGcReliabilityFramework(scenario)) {
2253 buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2254 buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2257 buildCommands += "tests\\runtest.cmd ${runtestArguments}"
2259 } // end if (!isBuildOnly)
2261 if (!doCoreFxTesting) {
2262 // Run the rest of the build
2263 // Build the mscorlib for the other OS's
2264 buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2265 buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2267 if (arch == 'x64') {
2268 buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2271 // Zip up the tests directory so that we don't use so much space/time copying
2272 // 10s of thousands of files around.
2273 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2275 if (!isJitStressScenario(scenario)) {
2276 // For Windows, pull full test results and test drops for x86/x64.
2277 // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2278 Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2281 if (scenario == 'jitdiff') {
2282 // retrieve jit-dasm output for base commit, and run jit-diff
2284 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2285 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2290 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2296 assert isArmWindowsScenario(scenario)
2300 if (doCoreFxTesting) {
2301 buildOpts += ' skiptests'
2303 buildOpts += " -priority=${priority}"
2306 // This is now a build only job. Do not run tests. Use the flow job.
2307 buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2309 if (doCoreFxTesting) {
2312 // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2313 def envScriptPath = ''
2314 def buildCommandsStr = ''
2315 envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2316 buildCommandsStr += envScriptCreate(os, envScriptPath)
2317 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2318 envScriptFinalize(os, envScriptPath)
2319 buildCommands += buildCommandsStr
2321 def workspaceRelativeFxRootLinux = "_/fx"
2322 def workspaceRelativeFxRootWin = "_\\fx"
2323 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2324 def fxBranch = getFxBranch(branch)
2326 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2328 // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2329 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2330 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2332 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2333 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2335 // Zip up the tests directory so that we don't use so much space/time copying
2336 // 10s of thousands of files around.
2337 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2340 Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2344 println("Unknown architecture: ${architecture}");
2349 // end case 'Windows_NT'; editor brace matching: }
2358 case 'Fedora24': // editor brace matching: {
2359 switch (architecture) {
2362 if (architecture == 'x86' && os == 'Ubuntu') {
2363 // build and PAL test
2364 def dockerImage = getDockerImageName(architecture, os, true)
2365 buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2366 dockerImage = getDockerImageName(architecture, os, false)
2367 buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2368 Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2369 Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2373 if (scenario == 'formatting') {
2374 buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2375 Utilities.addArchival(newJob, "format.patch", "", true, false)
2379 if (scenario == 'illink') {
2380 assert(os == 'Ubuntu')
2381 buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2384 if (!doCoreFxTesting) {
2385 // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2386 // only on supported OS platforms.
2387 def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2388 def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2390 buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2391 buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2393 // Basic archiving of the build
2394 Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2396 Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2399 if (scenario == 'corefx_innerloop') {
2400 assert os == 'Ubuntu' || 'OSX10.12'
2401 assert architecture == 'x64'
2403 buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2404 buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2405 buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2408 // Archive and process (only) the test results
2409 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml")
2410 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2413 // Corefx stress testing
2414 assert os == 'Ubuntu'
2415 assert architecture == 'x64'
2416 assert lowerConfiguration == 'checked'
2417 assert isJitStressScenario(scenario)
2420 buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2422 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2424 def envScriptCmds = envScriptCreate(os, scriptFileName)
2425 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2426 envScriptCmds += envScriptFinalize(os, scriptFileName)
2427 buildCommands += envScriptCmds
2429 // Build and text corefx
2430 def workspaceRelativeFxRoot = "_/fx"
2431 def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2432 def fxBranch = getFxBranch(branch)
2434 buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
2436 // Archive and process (only) the test results
2437 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2438 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
2443 // Emulator cross builds for ARM runs on Tizen currently
2444 assert os == 'Tizen'
2446 def arm_abi = "armel"
2447 def linuxCodeName = "tizen"
2449 // Unzip the Windows test binaries first. Exit with 0
2450 buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2452 // Unpack the corefx binaries
2453 buildCommands += "mkdir ./bin/CoreFxBinDir"
2454 buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2456 // Call the ARM CI script to cross build and test using docker
2457 buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2460 --linuxCodeName=${linuxCodeName} \\
2461 --buildConfig=${lowerConfiguration} \\
2462 --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2463 --coreFxBinDir=./bin/CoreFxBinDir \\
2464 --testDirFile=./tests/testsRunningInsideARM.txt"""
2466 // Basic archiving of the build, no pal tests
2467 Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2471 // Non-Windows ARM cross builds on hardware run on Ubuntu only
2472 assert (os == 'Ubuntu')
2474 // Add some useful information to the log file. Ignore return codes.
2475 buildCommands += "uname -a || true"
2477 def additionalOpts = ""
2478 if (architecture == 'arm') {
2479 additionalOpts = "-e CAC_ROOTFS_DIR=/crossrootfs/x86"
2482 // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2483 // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2484 // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2485 // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2486 // ZIP up the generated CoreFX runtime and tests.
2488 def dockerImage = getDockerImageName(architecture, os, true)
2489 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${additionalOpts} ${dockerImage} "
2491 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross crosscomponent"
2493 if (doCoreFxTesting) {
2494 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2496 def envScriptCmds = envScriptCreate(os, scriptFileName)
2497 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2498 envScriptCmds += envScriptFinalize(os, scriptFileName)
2499 buildCommands += envScriptCmds
2501 // Build and text corefx
2502 def workspaceRelativeFxRootLinux = "_/fx"
2503 def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2504 def fxBranch = getFxBranch(branch)
2506 buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2508 // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2509 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2510 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/bin/tests"
2512 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2513 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2514 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
2516 else if (isCrossGenComparisonScenario(scenario)) {
2517 buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2519 def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
2521 def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2522 def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2523 def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2524 def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2525 def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
2526 def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2528 def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2529 def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArchitecture}/crossgen"
2531 buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2532 buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2533 buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
2534 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeCrossResultDir}"
2536 Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2539 // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
2540 // download the appropriate CoreFX packages.
2541 // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
2542 // just to be consistent with the "build.sh" case -- so both are run with the same environment.
2544 buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2546 // ZIP up for the test job (created in the flow job code):
2547 // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root,
2548 // used by runtest.sh as the "--coreOverlayDir" argument.
2549 // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests,
2550 // used by runtest.sh as the "--testNativeBinDir" argument.
2552 // These commands are assumed to be run from the root of the workspace.
2553 buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2554 buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
2556 Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
2559 // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2560 // the cleanup task in Jenkins can't remove.
2563 azureVMAgentPostBuildAction {
2564 agentPostBuildAction('Delete agent after build execution (when idle).')
2570 println("Unknown architecture: ${architecture}");
2575 // editor brace matching: }
2577 println("Unknown os: ${os}");
2582 return buildCommands
2585 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2586 // Returns true if the job should be generated.
2587 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2589 // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
2590 if (scenario == 'innerloop' && !isPR) {
2594 // Run basic corefx tests only on PR-triggered jobs
2595 // Runs under Release and Checked
2596 if (scenario == 'corefx_innerloop' && !isPR) {
2600 // Tizen is only supported for armem architecture
2601 if (os == 'Tizen' && architecture != 'armem') {
2605 // Filter based on architecture.
2607 switch (architecture) {
2610 if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2615 if (os != 'Tizen') {
2619 case 'x86_arm_altjit':
2620 case 'x64_arm64_altjit':
2621 if (os != 'Windows_NT') {
2626 if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2631 // Everything implemented
2634 println("Unknown architecture: ${architecture}")
2639 // Which (Windows) build only jobs are required?
2641 def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2644 switch (architecture) {
2647 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2648 if (!isCoreFxScenario(scenario)) {
2654 if (!isNormalOrInnerloop) {
2663 // Filter based on scenario.
2665 if (isJitStressScenario(scenario)) {
2666 if (configuration != 'Checked') {
2670 def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
2675 switch (architecture) {
2677 case 'x86_arm_altjit':
2678 case 'x64_arm64_altjit':
2682 // x86 ubuntu: no stress modes
2683 if (os == 'Ubuntu') {
2689 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2690 // No "regular" Windows arm corefx jobs, e.g.
2691 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2692 // the Ubuntu arm "regular" jobs don't run tests anyway).
2693 if (os == 'Windows_NT') {
2694 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2699 if (!isCoreFxScenario(scenario)) {
2706 if (os == 'Windows_NT') {
2707 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2712 if (!isCoreFxScenario(scenario)) {
2719 // arm64: stress is handled through flow jobs.
2720 // armem: no stress jobs for ARM emulator.
2724 else if (isR2RScenario(scenario)) {
2725 if (os != 'Windows_NT') {
2729 if (isR2RBaselineScenario(scenario)) {
2730 // no need for Debug scenario; Checked is sufficient
2731 if (configuration != 'Checked' && configuration != 'Release') {
2735 else if (isR2RStressScenario(scenario)) {
2736 // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2737 if (configuration != 'Checked') {
2742 switch (architecture) {
2745 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2752 else if (isCrossGenComparisonScenario(scenario)) {
2753 return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2759 // The ilrt build isn't necessary except for Windows_NT2003. Non-Windows NT uses
2760 // the default scenario build
2761 if (os != 'Windows_NT') {
2765 if (architecture != 'x64') {
2769 if (configuration != 'Release') {
2774 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2777 if (architecture != 'x64') {
2780 if (configuration != 'Checked') {
2786 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2789 if (architecture != 'x64') {
2792 if (configuration != 'Release') {
2796 case 'gc_reliability_framework':
2797 case 'standalone_gc':
2798 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2802 if (architecture != 'x64') {
2806 if (configuration != 'Release' && configuration != 'Checked') {
2810 // We only run Windows and Ubuntu x64 Checked for formatting right now
2812 if (os != 'Windows_NT' && os != 'Ubuntu') {
2815 if (architecture != 'x64') {
2818 if (configuration != 'Checked') {
2823 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2826 if (architecture != 'x64' && architecture != 'x86') {
2834 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2838 case 'corefx_innerloop':
2839 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2842 if (architecture != 'x64') {
2847 println("Unknown scenario: ${scenario}")
2853 // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2854 // runtime behavior, not compile-time behavior, are not interesting.
2855 switch (architecture) {
2856 case 'x86_arm_altjit':
2857 case 'x64_arm64_altjit':
2858 if (isGCStressRelatedTesting(scenario)) {
2866 // The job was not filtered out, so we should generate it!
2870 Constants.allScenarios.each { scenario ->
2871 [true, false].each { isPR ->
2872 Constants.architectureList.each { architecture ->
2873 Constants.configurationList.each { configuration ->
2874 Constants.osList.each { os ->
2875 // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2876 // and reset the os to Windows_NT
2877 def isBuildOnly = false
2878 if (os == 'Windows_NT_BuildOnly') {
2883 if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2888 def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2889 def folderName = getJobFolder(scenario)
2891 // Create the new job
2892 def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2894 // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
2895 // "top-level" jobs. Build only jobs are such jobs.
2898 addToViews(newJob, isPR, architecture, os)
2901 setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2903 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2904 addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2905 setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2907 // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2908 // We don't use a flow job for this, but we do depend on there being existing builds with these
2909 // artifacts produced.
2910 if ((architecture == 'armem') && (os == 'Tizen')) {
2911 // Define the Windows Tests and Corefx build job names
2912 def lowerConfiguration = configuration.toLowerCase()
2913 def WindowsTestsName = projectFolder + '/' +
2914 Utilities.getFullJobName(project,
2915 getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2917 def fxBranch = getFxBranch(branch)
2918 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2919 Utilities.getFolderName(fxBranch)
2921 def arm_abi = 'armel'
2922 def corefx_os = 'tizen'
2924 // Let's use release CoreFX to test checked CoreCLR,
2925 // because we do not generate checked CoreFX in CoreFX CI yet.
2926 def corefx_lowerConfiguration = lowerConfiguration
2927 if (lowerConfiguration == 'checked') {
2928 corefx_lowerConfiguration = 'release'
2931 // Copy the Windows test binaries and the Corefx build binaries
2934 copyArtifacts(WindowsTestsName) {
2935 includePatterns('bin/tests/tests.zip')
2937 latestSuccessful(true)
2940 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2941 includePatterns('bin/build.tar.gz')
2943 latestSuccessful(true)
2950 def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2954 if (os == 'Windows_NT') {
2955 buildCommands.each { buildCommand ->
2956 batchFile(buildCommand)
2960 buildCommands.each { buildCommand ->
2973 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2974 // Returns the newly created job.
2975 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2977 def osGroup = getOSGroup(os)
2978 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2980 def jobFolder = getJobFolder(scenario)
2981 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2983 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2987 // Set up the copies
2989 // Coreclr build we are trying to test
2991 // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2993 copyArtifacts(inputCoreCLRBuildName) {
2994 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2996 buildNumber('${CORECLR_BUILD}')
3000 if (isCoreFxScenario(scenario)) {
3002 // Only arm/arm64 supported for corefx testing now.
3003 assert architecture == 'arm' || architecture == 'arm64'
3005 // Unzip CoreFx runtime
3006 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
3008 // Unzip CoreFx tests.
3009 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')\"")
3011 // Add the script to run the corefx tests
3012 def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
3013 def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\bin\\tests"
3014 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
3015 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
3017 } else { // !isCoreFxScenario(scenario)
3020 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3022 def buildCommands = ""
3024 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3025 def addEnvVariable = { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3026 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3028 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3029 addCommand("SETLOCAL ENABLEEXTENSIONS")
3032 addEnvVariable("CORE_ROOT", coreRootLocation)
3033 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3034 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3036 // If we are running a stress mode, we'll set those variables as well
3037 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3038 def stressValues = null
3039 if (isJitStressScenario(scenario)) {
3040 stressValues = Constants.jitStressModeScenarios[scenario]
3043 stressValues = Constants.r2rStressScenarios[scenario]
3046 stressValues.each { key, value ->
3047 addEnvVariable(key, value)
3051 if (isR2RScenario(scenario)) {
3052 // Crossgen the framework assemblies.
3053 buildCommands += """
3054 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3055 @goto skip_PrecompileAssembly
3058 @REM Skip mscorlib since it is already precompiled.
3059 @if /I "%3" == "mscorlib.dll" exit /b 0
3060 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3062 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3063 @if "%errorlevel%" == "-2146230517" (
3064 echo %2 is not a managed assembly.
3065 ) else if "%errorlevel%" == "-2146234344" (
3066 echo %2 is not a managed assembly.
3067 ) else if %errorlevel% neq 0 (
3068 echo Unable to precompile %2
3074 :skip_PrecompileAssembly
3077 // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3078 // crossgen on tests before running them.
3079 addEnvVariable("RunCrossGen", "true")
3080 } // isR2RScenario(scenario)
3082 // Create the smarty command
3083 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
3084 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
3085 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
3086 def addArchSpecificExclude = { architectureToExclude, exclude -> addExclude(exclude) }
3088 // Exclude tests based on scenario.
3089 Constants.validArmWindowsScenarios[scenario].each { excludeTag ->
3090 addArchSpecificExclude(architecture, excludeTag)
3093 if (isInnerloopTestScenario(scenario)) {
3097 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
3098 // TODO: We should create a "long running" job that runs these with a longer timeout.
3099 addExclude("LONG_RUNNING")
3101 smartyCommand += "/lstFile Tests.lst"
3103 def testListArch = [
3108 def archLocation = testListArch[architecture]
3110 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
3111 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
3112 addCommand("${smartyCommand}")
3114 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
3115 // However, we also need to remove all the variables that were set during this batch file, so we
3116 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
3117 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
3118 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
3119 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
3120 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
3121 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
3123 // ZIP up the smarty output, no matter what the smarty result.
3124 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
3126 addCommand("echo %errorlevel%")
3127 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
3129 // Use the smarty errorlevel as the script errorlevel.
3130 addCommand("exit /b %__save_smarty_errorlevel%")
3132 batchFile(buildCommands)
3133 } // non-corefx testing
3137 if (!isCoreFxScenario(scenario)) {
3138 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
3140 // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
3141 // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
3142 // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
3143 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
3149 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3150 // E.g., non-Windows tests.
3151 // Returns the newly created job.
3152 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3154 def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
3155 def doCoreFxTesting = isCoreFxScenario(scenario)
3157 def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3159 def osGroup = getOSGroup(os)
3160 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3163 def useServerGC = false
3165 // Enable Server GC for Ubuntu PR builds
3166 // REVIEW: why? Does this apply to all architectures? Why only PR?
3167 if (os == 'Ubuntu' && isPR) {
3168 testOpts += ' --useServerGC'
3172 if (isR2RScenario(scenario)) {
3174 testOpts += ' --crossgen --runcrossgentests'
3176 if (scenario == 'r2r_jitstress1') {
3177 testOpts += ' --jitstress=1'
3179 else if (scenario == 'r2r_jitstress2') {
3180 testOpts += ' --jitstress=2'
3182 else if (scenario == 'r2r_jitstress1_tiered') {
3183 testOpts += ' --jitstress=1'
3185 else if (scenario == 'r2r_jitstress2_tiered') {
3186 testOpts += ' --jitstress=2'
3188 else if (scenario == 'r2r_jitstressregs1') {
3189 testOpts += ' --jitstressregs=1'
3191 else if (scenario == 'r2r_jitstressregs2') {
3192 testOpts += ' --jitstressregs=2'
3194 else if (scenario == 'r2r_jitstressregs3') {
3195 testOpts += ' --jitstressregs=3'
3197 else if (scenario == 'r2r_jitstressregs4') {
3198 testOpts += ' --jitstressregs=4'
3200 else if (scenario == 'r2r_jitstressregs8') {
3201 testOpts += ' --jitstressregs=8'
3203 else if (scenario == 'r2r_jitstressregs0x10') {
3204 testOpts += ' --jitstressregs=0x10'
3206 else if (scenario == 'r2r_jitstressregs0x80') {
3207 testOpts += ' --jitstressregs=0x80'
3209 else if (scenario == 'r2r_jitstressregs0x1000') {
3210 testOpts += ' --jitstressregs=0x1000'
3212 else if (scenario == 'r2r_jitminopts') {
3213 testOpts += ' --jitminopts'
3215 else if (scenario == 'r2r_jitforcerelocs') {
3216 testOpts += ' --jitforcerelocs'
3218 else if (scenario == 'r2r_gcstress15') {
3219 testOpts += ' --gcstresslevel=0xF'
3222 else if (scenario == 'jitdiff') {
3223 testOpts += ' --jitdisasm --crossgen'
3225 else if (scenario == 'illink') {
3226 testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3228 else if (isLongGc(scenario)) {
3229 // Long GC tests behave very poorly when they are not
3230 // the only test running (many of them allocate until OOM).
3231 testOpts += ' --sequential'
3233 // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3234 // for running long GC and GCSimulator tests, respectively. We don't use them
3235 // here because using a playlist file produces much more readable output on the CI machines
3236 // and reduces running time.
3238 // The Long GC playlist contains all of the tests that are
3239 // going to be run. The GCSimulator playlist contains all of
3240 // the GC simulator tests.
3241 if (scenario == 'longgc') {
3242 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3244 else if (scenario == 'gcsimulator') {
3245 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3248 else if (isGcReliabilityFramework(scenario)) {
3249 testOpts += ' --build-overlay-only'
3251 else if (scenario == 'standalone_gc') {
3252 if (osGroup == 'OSX') {
3253 testOpts += ' --gcname=libclrgc.dylib'
3255 else if (osGroup == 'Linux') {
3256 testOpts += ' --gcname=libclrgc.so'
3259 println("Unexpected OS group: ${osGroup} for os ${os}")
3264 // The ARM Ubuntu corefx test job doesn't depend on a Windows test build, and hence inputTestsBuildName
3265 // will be null in this case.
3267 def jobFolder = getJobFolder(scenario)
3268 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3270 if (inputTestsBuildName != null) {
3271 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from')
3273 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3277 // Set up the copies
3279 // Coreclr build containing the tests and mscorlib
3280 // pri1 jobs still need to copy windows_nt built tests
3281 if (inputTestsBuildName != null) {
3282 copyArtifacts(inputTestsBuildName) {
3283 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3285 buildNumber('${CORECLR_WINDOWS_BUILD}')
3290 // Coreclr build we are trying to test
3292 // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
3294 copyArtifacts(inputCoreCLRBuildName) {
3295 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3297 buildNumber('${CORECLR_BUILD}')
3301 if (isUbuntuArmJob) {
3302 // Add some useful information to the log file. Ignore return codes.
3303 shell("uname -a || true")
3306 if (architecture == 'x86') {
3307 shell("mkdir ./bin/CoreFxNative")
3309 def fxBranch = getFxBranch(branch)
3310 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3312 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3313 includePatterns('bin/build.tar.gz')
3314 targetDirectory('bin/CoreFxNative')
3316 latestSuccessful(true)
3320 shell("mkdir ./bin/CoreFxBinDir")
3321 shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3324 // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3325 if (doCoreFxTesting) {
3326 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3327 shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3330 // Unzip the tests first. Exit with 0
3331 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
3332 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
3335 // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test
3336 // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example.
3337 // We should probably do the "generatelayoutonly" step on the build machine for all architectures.
3338 // However, it's believed that perhaps there's an issue with executable permission bits not getting
3339 // copied correctly.
3340 if (!doCoreFxTesting) {
3341 if (isUbuntuArmJob) {
3342 def lowerConfiguration = configuration.toLowerCase()
3343 shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
3344 shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
3347 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
3351 // Execute the tests
3352 def runDocker = isNeedDocker(architecture, os, false)
3353 def dockerPrefix = ""
3356 def dockerImage = getDockerImageName(architecture, os, false)
3357 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3358 dockerCmd = dockerPrefix + "${dockerImage} "
3361 // If we are running a stress mode, we'll set those variables first.
3362 // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3363 if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3364 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3365 def envScriptCmds = envScriptCreate(os, scriptFileName)
3366 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3367 envScriptCmds += envScriptFinalize(os, scriptFileName)
3368 shell("${envScriptCmds}")
3369 testOpts += " --test-env=${scriptFileName}"
3372 // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3373 // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3374 // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3375 // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3376 if (isGCStressRelatedTesting(scenario)) {
3377 if (architecture == 'x64') {
3378 shell('./init-tools.sh')
3382 if (doCoreFxTesting) {
3384 \${WORKSPACE}/${workspaceRelativeFxRootLinux}/run-test.sh --sequential --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/bin --configurationGroup Release""")
3387 def runScript = "${dockerCmd}./tests/runtest.sh"
3391 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3392 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3393 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
3394 --copyNativeTestBin --limitedDumpGeneration ${testOpts}""")
3397 if (isGcReliabilityFramework(scenario)) {
3398 // runtest.sh doesn't actually execute the reliability framework - do it here.
3401 dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3404 shell("export COMPlus_gcServer=1")
3408 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3413 // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3414 if (os in ['Ubuntu']) {
3415 SummaryBuilder summaries = new SummaryBuilder()
3416 summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3417 summaries.emit(newJob)
3420 if (doCoreFxTesting) {
3421 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3422 if ((os == "Ubuntu") && (architecture == 'arm')) {
3423 // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3425 // [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3427 // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3428 // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3431 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/bin/**/testResults.xml")
3435 Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3441 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3443 assert isCrossGenComparisonScenario(scenario)
3445 def osGroup = getOSGroup(os)
3446 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3448 def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
3449 def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
3450 def workspaceRelativeNativeResultDir = "_/${osGroup}.${architecture}_${architecture}.${configuration}"
3452 def jobFolder = getJobFolder(scenario)
3453 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3455 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3458 def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3461 copyArtifacts(inputCoreCLRBuildName) {
3462 includePatterns("${workspaceRelativeArtifactsArchive}")
3464 buildNumber('${CORECLR_BUILD}')
3468 shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3470 def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3471 def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3472 def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3473 def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3475 def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3476 def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3478 shell("mkdir -p ${workspaceRelativeNativeResultDir}")
3479 shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
3480 shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
3481 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}")
3485 Utilities.addArchival(newJob, "${workspaceRelativeCrossResultDir}/**")
3486 Utilities.addArchival(newJob, "${workspaceRelativeNativeResultDir}/**")
3491 // Create a test job that will be used by a flow job.
3492 // Returns the newly created job.
3493 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3494 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3495 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
3497 def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3500 if (windowsArmJob) {
3501 assert inputTestsBuildName == null
3502 newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3504 else if (isCrossGenComparisonScenario(scenario)) {
3505 assert inputTestsBuildName == null
3506 newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3509 newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3512 setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3514 if (scenario == 'jitdiff') {
3515 def osGroup = getOSGroup(os)
3516 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3519 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3520 setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3525 // Create a flow job to tie together a build job with the given test job.
3526 // The 'inputTestsBuildName' argument might be null if the flow job doesn't depend on a Windows build job.
3527 // Returns the new flow job.
3528 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName)
3530 // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3531 // Linux CoreCLR test
3532 def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3533 def jobFolder = getJobFolder(scenario)
3535 def newFlowJob = null
3537 if (inputTestsBuildName == null) {
3538 newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3540 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3542 // And then build the test build
3543 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3546 JobReport.Report.addReference(inputCoreCLRBuildName)
3547 JobReport.Report.addReference(fullTestJobName)
3550 newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3552 // Build the input jobs in parallel
3554 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
3555 { windowsBuildJob = build(params, '${inputTestsBuildName}') }
3558 // And then build the test build
3559 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
3560 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
3563 JobReport.Report.addReference(inputCoreCLRBuildName)
3564 JobReport.Report.addReference(inputTestsBuildName)
3565 JobReport.Report.addReference(fullTestJobName)
3568 addToViews(newFlowJob, isPR, architecture, os)
3570 setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3572 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3573 addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3578 // Determine if we should generate a flow job for the given parameters.
3579 // Returns true if the job should be generated.
3580 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3582 // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
3583 if (scenario == 'innerloop' && !isPR) {
3587 if (scenario == 'corefx_innerloop') {
3591 // Filter based on OS and architecture.
3593 switch (architecture) {
3596 if (os != "Ubuntu" && os != "Windows_NT") {
3601 if (os != "Ubuntu") {
3606 if (!(os in Constants.crossList)) {
3609 if (os == "Windows_NT") {
3614 case 'x86_arm_altjit':
3615 case 'x64_arm64_altjit':
3619 println("Unknown architecture: ${architecture}")
3624 def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3626 // Filter based on scenario in OS.
3628 if (os == 'Windows_NT') {
3629 assert architecture == 'arm' || architecture == 'arm64'
3630 if (!isArmWindowsScenario(scenario)) {
3633 if (isNormalOrInnerloop && (configuration == 'Debug')) {
3634 // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3635 // To do that, it doesn't require a flow job.
3641 if (architecture == 'arm64') {
3642 if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3646 if (isNormalOrInnerloop && (configuration == 'Debug')) {
3647 // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3648 // To do that, it doesn't require a flow job.
3652 else if (architecture == 'arm') {
3653 if (!(scenario in Constants.validLinuxArmScenarios)) {
3657 else if (architecture == 'x86') {
3658 // Linux/x86 only want innerloop and default test
3659 if (!isNormalOrInnerloop) {
3663 else if (architecture == 'x64') {
3664 // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3665 // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3666 // build from a Windows machine.
3667 if (isCoreFxScenario(scenario)) {
3673 // For CentOS, we only want Checked/Release builds.
3674 if (os == 'CentOS7.1') {
3675 if (configuration != 'Checked' && configuration != 'Release') {
3678 if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3683 // For RedHat and Debian, we only do Release builds.
3684 else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3685 if (configuration != 'Release') {
3688 if (!isNormalOrInnerloop) {
3693 // Next, filter based on scenario.
3695 if (isJitStressScenario(scenario)) {
3696 if (configuration != 'Checked') {
3700 else if (isR2RBaselineScenario(scenario)) {
3701 if (configuration != 'Checked' && configuration != 'Release') {
3705 else if (isR2RStressScenario(scenario)) {
3706 if (configuration != 'Checked') {
3710 else if (isCrossGenComparisonScenario(scenario)) {
3711 return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3719 // Long GC tests take a long time on non-Release builds
3720 // ilrt is also Release only
3721 if (configuration != 'Release') {
3727 if (configuration != 'Checked') {
3732 case 'gc_reliability_framework':
3733 case 'standalone_gc':
3734 if (configuration != 'Release' && configuration != 'Checked') {
3743 if (os != 'Windows_NT' && os != 'Ubuntu') {
3753 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3759 println("Unknown scenario: ${scenario}")
3765 // The job was not filtered out, so we should generate it!
3769 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3770 Constants.allScenarios.each { scenario ->
3771 [true, false].each { isPR ->
3772 Constants.architectureList.each { architecture ->
3773 Constants.configurationList.each { configuration ->
3774 Constants.osList.each { os ->
3776 if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3780 def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3781 def doCoreFxTesting = isCoreFxScenario(scenario)
3782 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3784 // Figure out the job name of the CoreCLR build the test will depend on.
3786 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3787 def inputCoreCLRBuildIsBuildOnly = false
3788 if (doCoreFxTesting) {
3789 // Every CoreFx test depends on its own unique build.
3790 inputCoreCLRBuildScenario = scenario
3791 if (windowsArmJob) {
3792 // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3793 inputCoreCLRBuildIsBuildOnly = true
3796 if (doCrossGenComparison) {
3797 inputCoreCLRBuildScenario = scenario
3800 def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3801 def inputCoreCLRBuildName = projectFolder + '/' +
3802 Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3804 // Figure out the name of the build job that the test job will depend on.
3805 // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other
3806 // tests (e.g., Linux ARM), we depend on a Windows build to get the tests.
3807 // For CoreFX tests, however, Linux doesn't need the Windows build for the tests, since the
3808 // CoreFX build creates the tests.
3810 def inputTestsBuildName = null
3812 if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison) {
3813 def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3815 def inputTestsBuildArch = architecture
3816 if (architecture == "arm64") {
3817 // Use the x64 test build for arm64 unix
3818 inputTestsBuildArch = "x64"
3820 else if (architecture == "arm") {
3821 // Use the x86 test build for arm unix
3822 inputTestsBuildArch = "x86"
3825 def inputTestsBuildIsBuildOnly = true
3827 inputTestsBuildName = projectFolder + '/' +
3828 Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR)
3831 // =============================================================================================
3832 // Create the test job
3833 // =============================================================================================
3835 def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
3837 // =============================================================================================
3838 // Create a build flow to join together the build and tests required to run this test.
3839 // =============================================================================================
3841 if (os == 'RHEL7.2' || os == 'Debian8.4') {
3842 // Do not create the flow job for RHEL jobs.
3846 def fullTestJobName = projectFolder + '/' + testJob.name
3847 def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName)
3855 JobReport.Report.generateJobReport(out)
3857 // Make the call to generate the help job
3858 Utilities.createHelperJob(this, project, branch,
3859 "Welcome to the ${project} Repository", // This is prepended to the help message
3860 "Have a nice day!") // This is appended to the help message. You might put known issues here.
3862 Utilities.addCROSSCheck(this, project, branch)