1 // Import the utility functionality.
3 import jobs.generation.*
5 // The input project name (e.g. dotnet/coreclr)
6 def project = GithubProject
7 // The input branch name (e.g. master)
8 def branch = GithubBranchName
9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
11 // Create a folder for JIT stress jobs and associated folder views
13 Utilities.addStandardFolderView(this, 'jitstress', project)
15 // Create a folder for testing via illink
17 Utilities.addStandardFolderView(this, 'illink', project)
19 def static getOSGroup(def os) {
20 def osGroupMap = ['Ubuntu' : 'Linux',
21 'Ubuntu16.04' : 'Linux',
22 'Ubuntu16.10' : 'Linux',
24 'Debian8.4' : 'Linux',
26 'CentOS7.1' : 'Linux',
29 'Windows_NT' : 'Windows_NT']
30 def osGroup = osGroupMap.get(os, null)
31 assert osGroup != null : "Could not find os group for ${os}"
35 // We use this class (vs variables) so that the static functions can access data here.
38 // We have very limited Windows ARM64 hardware (used for ARM/ARM64 testing) and Linux/arm32 and Linux/arm64 hardware.
39 // So only allow certain branches to use it.
40 def static LimitedHardwareBranches = [
43 // Innerloop build OS's
44 // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
45 // test execution in the build flow runs. It generates the exact same build
46 // as Windows_NT but without running the tests.
52 'Windows_NT_BuildOnly',
60 def static crossList = [
68 // This is a set of JIT stress modes combined with the set of variables that
69 // need to be set to actually enable that stress mode. The key of the map is the stress mode and
70 // the values are the environment variables
71 def static jitStressModeScenarios = [
72 'minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
73 'tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
74 'no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'],
75 'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'],
76 'forcerelocs' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'],
77 'jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
78 'jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
79 'jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'],
80 'jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'],
81 'jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
82 'jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
83 'jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
84 'jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
85 'jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
86 'jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
87 'jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
88 'jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
89 'jitstress2_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
90 'jitstress2_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
91 'jitstress2_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
92 'jitstress2_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
93 'jitstress2_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
94 'jitstress2_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
95 'jitstress2_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
96 'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
97 'tailcallstress' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'],
98 'jitsse2only' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
99 'jitnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'],
100 'jitincompletehwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'],
101 'jitx86hwintrinsicnoavx' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding
102 'jitx86hwintrinsicnoavx2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB
103 'jitx86hwintrinsicnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD
104 'jitnox86hwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'],
105 'corefx_baseline' : ['COMPlus_TieredCompilation' : '0'], // corefx baseline
106 'corefx_minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'],
107 'corefx_tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default
108 'corefx_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'],
109 'corefx_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'],
110 'corefx_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'],
111 'corefx_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'],
112 'corefx_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'],
113 'corefx_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'],
114 'corefx_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'],
115 'corefx_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'],
116 'corefx_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'],
117 'corefx_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'],
118 'gcstress0x3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'],
119 'gcstress0xc' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'],
120 'zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
121 'heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'],
122 'gcstress0xc_zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
123 'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress' : '2'],
124 'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
125 'gcstress0xc_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '1'],
126 'gcstress0xc_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '2'],
127 'gcstress0xc_minopts_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
130 // This is a set of ReadyToRun stress scenarios
131 def static r2rStressScenarios = [
132 'r2r_jitstress1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"],
133 'r2r_jitstress2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"],
134 'r2r_jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"],
135 'r2r_jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"],
136 'r2r_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"],
137 'r2r_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"],
138 'r2r_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"],
139 'r2r_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"],
140 'r2r_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"],
141 'r2r_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"],
142 'r2r_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"],
143 'r2r_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"],
144 'r2r_jitminopts' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"],
145 'r2r_jitforcerelocs' : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"],
146 'r2r_gcstress15' : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"],
147 'r2r_no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'],
150 // This is the basic set of scenarios
151 def static basicScenarios = [
159 // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work
161 'gc_reliability_framework',
164 'crossgen_comparison',
167 def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
169 // Valid PR trigger combinations.
170 def static prTriggeredValidInnerLoopCombos = [
179 'Windows_NT_BuildOnly': [
186 // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name
187 // to Tests.lst file categories to exclude.
189 // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
190 // and with the same values, with some commented out, for easier maintenance.
192 // Note that some scenarios that are commented out should be enabled, but haven't yet been.
194 def static validArmWindowsScenarios = [
204 // 'gc_reliability_framework'
206 // 'corefx_innerloop'
207 // 'crossgen_comparison'
211 'r2r_jitstress1_tiered',
212 'r2r_jitstress2_tiered',
213 'r2r_jitstressregs1',
214 'r2r_jitstressregs2',
215 'r2r_jitstressregs3',
216 'r2r_jitstressregs4',
217 'r2r_jitstressregs8',
218 'r2r_jitstressregs0x10',
219 'r2r_jitstressregs0x80',
220 'r2r_jitstressregs0x1000',
222 'r2r_jitforcerelocs',
224 'r2r_no_tiered_compilation',
227 'no_tiered_compilation',
228 'no_tiered_compilation_innerloop',
241 'jitstressregs0x1000',
242 'jitstress2_jitstressregs1',
243 'jitstress2_jitstressregs2',
244 'jitstress2_jitstressregs3',
245 'jitstress2_jitstressregs4',
246 'jitstress2_jitstressregs8',
247 'jitstress2_jitstressregs0x10',
248 'jitstress2_jitstressregs0x80',
249 'jitstress2_jitstressregs0x1000',
251 // 'jitsse2only' // Only relevant to xarch
252 'jitnosimd', // Only interesting on platforms where SIMD support exists.
253 // 'jitincompletehwintrinsic'
254 // 'jitx86hwintrinsicnoavx'
255 // 'jitx86hwintrinsicnoavx2'
256 // 'jitx86hwintrinsicnosimd'
257 // 'jitnox86hwintrinsic'
258 'corefx_baseline', // corefx tests don't use smarty
259 'corefx_minopts', // corefx tests don't use smarty
260 'corefx_tieredcompilation', // corefx tests don't use smarty
261 'corefx_jitstress1', // corefx tests don't use smarty
262 'corefx_jitstress2', // corefx tests don't use smarty
263 'corefx_jitstressregs1', // corefx tests don't use smarty
264 'corefx_jitstressregs2', // corefx tests don't use smarty
265 'corefx_jitstressregs3', // corefx tests don't use smarty
266 'corefx_jitstressregs4', // corefx tests don't use smarty
267 'corefx_jitstressregs8', // corefx tests don't use smarty
268 'corefx_jitstressregs0x10', // corefx tests don't use smarty
269 'corefx_jitstressregs0x80', // corefx tests don't use smarty
270 'corefx_jitstressregs0x1000', // corefx tests don't use smarty
275 'gcstress0xc_zapdisable',
276 'gcstress0xc_zapdisable_jitstress2',
277 'gcstress0xc_zapdisable_heapverify1',
278 'gcstress0xc_jitstress1',
279 'gcstress0xc_jitstress2',
280 'gcstress0xc_minopts_heapverify1',
283 // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
286 'minopts_zapdisable',
287 'gcstress0x3_jitstress1',
288 'gcstress0x3_jitstress2',
289 'gcstress0x3_jitstressregs1',
290 'gcstress0x3_jitstressregs2',
291 'gcstress0x3_jitstressregs3',
292 'gcstress0x3_jitstressregs4',
293 'gcstress0x3_jitstressregs8',
294 'gcstress0x3_jitstressregs0x10',
295 'gcstress0x3_jitstressregs0x80',
296 'gcstress0x3_jitstressregs0x1000',
297 'gcstress0xc_jitstressregs1',
298 'gcstress0xc_jitstressregs2',
299 'gcstress0xc_jitstressregs3',
300 'gcstress0xc_jitstressregs4',
301 'gcstress0xc_jitstressregs8',
302 'gcstress0xc_jitstressregs0x10',
303 'gcstress0xc_jitstressregs0x80',
304 'gcstress0xc_jitstressregs0x1000'
307 def static validLinuxArmScenarios = [
317 // 'gc_reliability_framework'
319 // 'corefx_innerloop'
320 'crossgen_comparison',
324 'r2r_jitstress1_tiered',
325 'r2r_jitstress2_tiered',
326 'r2r_jitstressregs1',
327 'r2r_jitstressregs2',
328 'r2r_jitstressregs3',
329 'r2r_jitstressregs4',
330 'r2r_jitstressregs8',
331 'r2r_jitstressregs0x10',
332 'r2r_jitstressregs0x80',
333 'r2r_jitstressregs0x1000',
335 'r2r_jitforcerelocs',
337 'r2r_no_tiered_compilation',
340 'no_tiered_compilation',
341 'no_tiered_compilation_innerloop',
354 'jitstressregs0x1000',
355 'jitstress2_jitstressregs1',
356 'jitstress2_jitstressregs2',
357 'jitstress2_jitstressregs3',
358 'jitstress2_jitstressregs4',
359 'jitstress2_jitstressregs8',
360 'jitstress2_jitstressregs0x10',
361 'jitstress2_jitstressregs0x80',
362 'jitstress2_jitstressregs0x1000',
364 // 'jitsse2only' // Only relevant to xarch
365 // 'jitnosimd' // Only interesting on platforms where SIMD support exists.
366 // 'jitincompletehwintrinsic'
367 // 'jitx86hwintrinsicnoavx'
368 // 'jitx86hwintrinsicnoavx2'
369 // 'jitx86hwintrinsicnosimd'
370 // 'jitnox86hwintrinsic'
373 'corefx_tieredcompilation',
376 'corefx_jitstressregs1',
377 'corefx_jitstressregs2',
378 'corefx_jitstressregs3',
379 'corefx_jitstressregs4',
380 'corefx_jitstressregs8',
381 'corefx_jitstressregs0x10',
382 'corefx_jitstressregs0x80',
383 'corefx_jitstressregs0x1000',
388 'gcstress0xc_zapdisable',
389 'gcstress0xc_zapdisable_jitstress2',
390 'gcstress0xc_zapdisable_heapverify1',
391 'gcstress0xc_jitstress1',
392 'gcstress0xc_jitstress2',
393 'gcstress0xc_minopts_heapverify1'
396 def static validLinuxArm64Scenarios = [
406 // 'gc_reliability_framework'
408 // 'corefx_innerloop'
409 'crossgen_comparison',
413 'r2r_jitstress1_tiered',
414 'r2r_jitstress2_tiered',
415 'r2r_jitstressregs1',
416 'r2r_jitstressregs2',
417 'r2r_jitstressregs3',
418 'r2r_jitstressregs4',
419 'r2r_jitstressregs8',
420 'r2r_jitstressregs0x10',
421 'r2r_jitstressregs0x80',
422 'r2r_jitstressregs0x1000',
424 'r2r_jitforcerelocs',
426 'r2r_no_tiered_compilation',
429 'no_tiered_compilation',
430 'no_tiered_compilation_innerloop',
443 'jitstressregs0x1000',
444 'jitstress2_jitstressregs1',
445 'jitstress2_jitstressregs2',
446 'jitstress2_jitstressregs3',
447 'jitstress2_jitstressregs4',
448 'jitstress2_jitstressregs8',
449 'jitstress2_jitstressregs0x10',
450 'jitstress2_jitstressregs0x80',
451 'jitstress2_jitstressregs0x1000',
453 // 'jitsse2only' // Only relevant to xarch
454 'jitnosimd', // Only interesting on platforms where SIMD support exists.
455 // 'jitincompletehwintrinsic'
456 // 'jitx86hwintrinsicnoavx'
457 // 'jitx86hwintrinsicnoavx2'
458 // 'jitx86hwintrinsicnosimd'
459 // 'jitnox86hwintrinsic'
462 'corefx_tieredcompilation',
465 'corefx_jitstressregs1',
466 'corefx_jitstressregs2',
467 'corefx_jitstressregs3',
468 'corefx_jitstressregs4',
469 'corefx_jitstressregs8',
470 'corefx_jitstressregs0x10',
471 'corefx_jitstressregs0x80',
472 'corefx_jitstressregs0x1000',
477 'gcstress0xc_zapdisable',
478 'gcstress0xc_zapdisable_jitstress2',
479 'gcstress0xc_zapdisable_heapverify1',
480 'gcstress0xc_jitstress1',
481 'gcstress0xc_jitstress2',
482 'gcstress0xc_minopts_heapverify1'
485 def static configurationList = ['Debug', 'Checked', 'Release']
487 // This is the set of architectures
488 // Some of these are pseudo-architectures:
489 // armem -- ARM builds/runs using an emulator. Used for Tizen runs.
490 def static architectureList = ['arm', 'armem', 'arm64', 'x64', 'x86']
492 // This set of architectures that cross build on Windows and run on Windows ARM64 hardware.
493 def static armWindowsCrossArchitectureList = ['arm', 'arm64']
496 // **************************************************************
497 // Create some specific views
499 // These aren't using the Utilities.addStandardFolderView() function, because that creates
500 // views based on a single regular expression. These views will be generated by adding a
501 // specific set of jobs to them.
503 // Utilities.addStandardFolderView() also creates a lot of additional stuff around the
504 // view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined
505 // those are required, don't add them (which simplifies the view pages, as well).
506 // **************************************************************
509 def static MergeJobView = null
510 def static PeriodicJobView = null
511 def static ArchitectureViews = [:]
512 def static OSViews = [:]
515 // MergeJobView: include all jobs that execute when a PR change is merged.
516 Views.MergeJobView = listView('Merge') {
529 // PeriodicJobView: include all jobs that execute on a schedule
530 Views.PeriodicJobView = listView('Periodic') {
543 // Create a view for non-PR jobs for each architecture.
544 Constants.architectureList.each { architecture ->
545 Views.ArchitectureViews[architecture] = listView(architecture) {
559 // Create a view for non-PR jobs for each OS.
560 Constants.osList.each { os ->
561 // Don't create one for the special 'Windows_NT_BuildOnly'
562 if (os == 'Windows_NT_BuildOnly') {
565 Views.OSViews[os] = listView(os) {
579 def static addToMergeView(def job) {
580 Views.MergeJobView.with {
587 def static addToPeriodicView(def job) {
588 Views.PeriodicJobView.with {
595 def static addToViews(def job, def isFlowJob, def isPR, def architecture, def os, def configuration, def scenario) {
597 // No views want PR jobs currently.
601 // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the
602 // "top-level" jobs. Build only jobs are such jobs.
603 if (os == 'Windows_NT_BuildOnly') {
608 // For non-flow jobs, which ones are only used by flow jobs?
609 if ((architecture == 'arm') || (architecture == 'arm64')) {
610 if (isCoreFxScenario(scenario)) {
611 // We have corefx-specific scenario builds for each of the runs, but these are driven by flow jobs.
615 // We're left with the basic normal/innerloop builds. We might want to remove these from the views also, if desired.
616 // However, there are a few, like the Debug Build, that is build only, not "Build and Test", that we should leave.
620 // Add to architecture view.
621 Views.ArchitectureViews[architecture].with {
628 Views.OSViews[os].with {
635 def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) {
636 addToPeriodicView(job)
637 Utilities.addPeriodicTrigger(job, cronString, alwaysRuns)
640 def static addGithubPushTriggerHelper(def job) {
641 // Disable all Push trigger jobs. All jobs will need to be requested.
642 // addToMergeView(job)
643 // Utilities.addGithubPushTrigger(job)
647 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
648 assert os instanceof String
649 assert architecture instanceof String
651 def armArches = ['arm', 'armem', 'arm64']
653 if (!(architecture in armArches)) {
654 assert options == null
655 Utilities.setMachineAffinity(job, os, 'latest-or-auto')
660 // This is an arm(64) job.
662 // There are several options.
666 // Arm32 (Build) -> latest-arm64
667 // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true
668 // Arm32 (Test) -> arm64-windows_nt
669 // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false
671 // Arm64 (Build) -> latest-arm64
672 // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
673 // Arm64 (Test) -> arm64-windows_nt
674 // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
678 // Arm32 emulator (Build, Test) -> arm-cross-latest
679 // |-> os == "Tizen" && (architecture == "armem")
681 // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware)
682 // |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
683 // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
684 // |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
685 // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
686 // |-> os == "Ubuntu" && (architecture == "arm")
688 // Arm64 (Build) -> arm64-cross-latest
689 // |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
690 // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Iron.Open queue
691 // |-> os != "Windows_NT" && architecture == "arm64"
693 // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
694 // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
696 // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
697 // we are using public VS2017 arm/arm64 tools in a VM from Helix.
699 // This has to be a arm arch
700 assert architecture in armArches
701 if (os == "Windows_NT") {
702 // arm32/arm64 Windows jobs share the same machines for now
703 def isBuild = options['use_arm64_build_machine'] == true
705 if (isBuild == true) {
707 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
710 Utilities.setMachineAffinity(job, 'windows.10.arm64.open')
713 assert os != 'Windows_NT'
715 if (architecture == 'armem') {
716 // arm emulator (Tizen). Build and test on same machine,
719 Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
722 // arm/arm64 Ubuntu on hardware.
723 assert architecture == 'arm' || architecture == 'arm64'
724 def isFlow = (options != null) && (options['is_flow_job'] == true)
725 def isBuild = (options != null) && (options['is_build_job'] == true)
726 if (isFlow || isBuild) {
727 // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
728 // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
729 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
731 // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
732 if (architecture == 'arm64') {
733 assert os == 'Ubuntu16.04'
735 label('Ubuntu.1604.Arm64.Iron.Open')
739 assert os == 'Ubuntu'
741 label('ubuntu.1404.arm32.open')
749 // setJobMachineAffinity: compute the machine affinity options for a job,
750 // then set the job with those affinity options.
751 def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job)
753 assert (isBuildJob && !isTestJob && !isFlowJob) ||
754 (!isBuildJob && isTestJob && !isFlowJob) ||
755 (!isBuildJob && !isTestJob && isFlowJob)
757 def affinityOptions = null
758 def affinityArchitecture = architecture
760 if (os == "Windows_NT") {
761 if (architecture in Constants.armWindowsCrossArchitectureList) {
763 affinityOptions = [ "use_arm64_build_machine" : true ]
764 } else if (isTestJob) {
765 affinityOptions = [ "use_arm64_build_machine" : false ]
766 } else if (isFlowJob) {
767 // For the flow jobs set the machine affinity as x64
768 affinityArchitecture = 'x64'
773 if ((architecture == 'arm64') || (architecture == 'arm')) {
775 affinityOptions = ['is_build_job': true]
776 } else if (isFlowJob) {
777 affinityOptions = ['is_flow_job': true]
782 setMachineAffinity(job, os, affinityArchitecture, affinityOptions)
785 def static isGCStressRelatedTesting(def scenario) {
786 // The 'r2r_gcstress15' scenario is a basic scenario.
787 // Detect it and make it a GCStress related.
788 if (scenario == 'r2r_gcstress15')
793 def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
794 def scenarioName = scenario.toLowerCase()
795 def isGCStressTesting = false
796 Constants.jitStressModeScenarios[scenario].each{ k, v ->
797 if (k in gcStressTestEnvVars) {
798 isGCStressTesting = true;
801 return isGCStressTesting
804 def static isCoreFxScenario(def scenario) {
805 def corefx_prefix = 'corefx_'
806 if (scenario.length() < corefx_prefix.length()) {
809 return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
812 def static isR2RBaselineScenario(def scenario) {
813 return (scenario == 'r2r')
816 def static isR2RStressScenario(def scenario) {
817 return Constants.r2rStressScenarios.containsKey(scenario)
820 def static isR2RScenario(def scenario) {
821 return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
824 def static isJitStressScenario(def scenario) {
825 return Constants.jitStressModeScenarios.containsKey(scenario)
828 def static isLongGc(def scenario) {
829 return (scenario == 'longgc' || scenario == 'gcsimulator')
832 def static isJitDiff(def scenario) {
833 return (scenario == 'jitdiff')
836 def static isGcReliabilityFramework(def scenario) {
837 return (scenario == 'gc_reliability_framework')
840 def static isArmWindowsScenario(def scenario) {
841 return Constants.validArmWindowsScenarios.contains(scenario)
844 def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) {
845 if (isBuildOnly == true) {
846 os = 'Windows_NT_BuildOnly'
849 def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os]
850 if (validOsPrTriggerArchConfigs != null) {
851 def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture]
852 if (validOsPrTriggerConfigs != null) {
853 if (configuration in validOsPrTriggerConfigs) {
862 // This means the job builds and runs the 'innerloop' test set. This does not mean the job is
863 // scheduled with a default PR trigger despite the correlation being true at the moment.
864 def static isInnerloopTestScenario(def scenario) {
865 return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
868 def static isCrossGenComparisonScenario(def scenario) {
869 return (scenario == 'crossgen_comparison')
872 def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
873 assert isCrossGenComparisonScenario(scenario)
874 return ((os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu16.04' && architecture == 'arm64')) && (configuration == 'Checked' || configuration == 'Release')
877 def static getFxBranch(def branch) {
878 def fxBranch = branch
879 // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
880 // branch even though CoreFX doesn't have such a branch.
881 if (branch == 'dev/unix_test_workflow') {
887 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
888 // 2 hours (120 minutes) is the default timeout
891 if (!isInnerloopTestScenario(scenario)) {
892 // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
897 // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
898 if (isGCStressRelatedTesting(scenario)) {
901 else if (isCoreFxScenario(scenario)) {
904 else if (isJitStressScenario(scenario)) {
907 else if (isR2RBaselineScenario(scenario)) {
910 else if (isLongGc(scenario)) {
913 else if (isJitDiff(scenario)) {
916 else if (isGcReliabilityFramework(scenario)) {
919 else if (architecture == 'armem' || architecture == 'arm64') {
923 if (architecture == 'arm') {
924 // ARM32 machines are particularly slow.
929 if (configuration == 'Debug') {
930 // Debug runs can be very slow. Add an hour.
934 // If we've changed the timeout from the default, set it in the job.
936 if (timeout != 120) {
937 Utilities.setJobTimeout(newJob, timeout)
941 def static getJobFolder(def scenario) {
942 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
945 if (scenario == 'illink') {
951 def static getStressModeDisplayName(def scenario) {
953 Constants.jitStressModeScenarios[scenario].each{ k, v ->
954 def prefixLength = 'COMPlus_'.length()
955 if (k.length() >= prefixLength) {
956 def modeName = k.substring(prefixLength, k.length())
957 if (displayStr != '') {
958 // Separate multiple variables with a space.
961 displayStr += modeName + '=' + v
965 if (isCoreFxScenario(scenario)) {
966 displayStr = ('CoreFx ' + displayStr).trim()
972 def static getR2RDisplayName(def scenario) {
973 // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
974 def displayStr = scenario
975 def prefixLength = 'r2r_'.length()
976 if (displayStr.length() >= prefixLength) {
977 displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
978 } else if (scenario == 'r2r') {
984 def static getScenarioDisplayString(def scenario) {
987 return "Innerloop Build and Test"
989 case 'no_tiered_compilation_innerloop':
990 def displayStr = getStressModeDisplayName(scenario)
991 return "Innerloop Build and Test (Jit - ${displayStr})"
993 case 'corefx_innerloop':
994 return "CoreFX Tests"
997 return "Build and Test"
1000 return "Jit Diff Build and Test"
1003 return "IL RoundTrip Build and Test"
1006 return "Long-Running GC Build & Test"
1009 return "GC Simulator"
1011 case 'standalone_gc':
1012 return "Standalone GC"
1014 case 'gc_reliability_framework':
1015 return "GC Reliability Framework"
1021 if (isJitStressScenario(scenario)) {
1022 def displayStr = getStressModeDisplayName(scenario)
1023 return "Build and Test (Jit - ${displayStr})"
1025 else if (isR2RScenario(scenario)) {
1026 def displayStr = getR2RDisplayName(scenario)
1027 return "${displayStr} Build and Test"
1030 return "${scenario}"
1035 println("Unknown scenario: ${scenario}");
1040 // Functions to create an environment script.
1041 // envScriptCreate -- initialize the script (call first)
1042 // envScriptFinalize -- finalize the script (call last)
1043 // envScriptSetStressModeVariables -- set stress mode variables in the env script
1044 // envScriptAppendExistingScript -- append an existing script to the generated script
1046 // Each script returns a string of commands. Concatenate all the strings together before
1047 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
1050 // Initialize the environment setting script.
1051 def static envScriptCreate(def os, def stepScriptLocation) {
1053 if (os == 'Windows_NT') {
1054 stepScript += "echo Creating TestEnv script\r\n"
1055 stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
1057 // Create at least an empty script.
1058 stepScript += "echo. > ${stepScriptLocation}\r\n"
1061 stepScript += "echo Creating environment setting script\n"
1062 stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
1068 // Generates the string for setting stress mode variables.
1069 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
1071 if (os == 'Windows_NT') {
1072 stressModeVars.each{ k, v ->
1073 // Write out what we are writing to the script file
1074 stepScript += "echo Setting ${k}=${v}\r\n"
1075 // Write out the set itself to the script file`
1076 stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
1080 stressModeVars.each{ k, v ->
1081 // Write out what we are writing to the script file
1082 stepScript += "echo Setting ${k}=${v}\n"
1083 // Write out the set itself to the script file`
1084 stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
1091 // Append an existing script to an environment script.
1092 // Returns string of commands to do this.
1093 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
1094 assert (os == 'Windows_NT')
1097 stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
1098 stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
1103 // Finalize an environment setting script.
1104 // Returns string of commands to do this.
1105 def static envScriptFinalize(def os, def stepScriptLocation) {
1108 if (os == 'Windows_NT') {
1109 // Display the resulting script. This is useful when looking at the output log file.
1110 stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
1111 stepScript += "type ${stepScriptLocation}\r\n"
1114 stepScript += "chmod +x ${stepScriptLocation}\n"
1120 def static isNeedDocker(def architecture, def os, def isBuild) {
1122 if (architecture == 'x86' && os == 'Ubuntu') {
1125 else if (architecture == 'armem') {
1128 else if (architecture == 'arm') {
1129 if (os == 'Ubuntu') {
1133 else if (architecture == 'arm64') {
1134 if (os == 'Ubuntu16.04') {
1140 if (architecture == 'x86' && os == 'Ubuntu') {
1147 def static getDockerImageName(def architecture, def os, def isBuild) {
1148 // We must change some docker private images to official later
1150 if (architecture == 'x86' && os == 'Ubuntu') {
1151 return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
1153 else if (architecture == 'armem') {
1154 if (os == 'Tizen') {
1155 return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1"
1158 else if (architecture == 'arm') {
1159 if (os == 'Ubuntu') {
1160 return "mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-cross-e435274-20180426002420"
1163 else if (architecture == 'arm64') {
1164 if (os == 'Ubuntu16.04') {
1165 return "mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
1170 if (architecture == 'x86' && os == 'Ubuntu') {
1171 return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
1174 println("Unknown architecture to use docker: ${architecture} ${os}");
1178 def static getTestArtifactsTgzFileName(def osGroup, def architecture, def configuration) {
1179 return "bin-tests-${osGroup}.${architecture}.${configuration}.tgz"
1182 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
1183 // and only allowing using this hardware in some specific branches.
1184 def static jobRequiresLimitedHardware(def architecture, def os) {
1185 if (architecture == 'arm') {
1186 // arm Windows and Linux hardware is limited.
1189 else if (architecture == 'arm64') {
1190 // arm64 Windows and Linux hardware is limited.
1198 // Calculates the name of the build job based on some typical parameters.
1200 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
1201 // If the architecture is x64, do not add that info into the build name.
1202 // Need to change around some systems and other builds to pick up the right builds
1205 def suffix = scenario != 'normal' ? "_${scenario}" : '';
1210 switch (architecture) {
1212 if (scenario == 'normal') {
1213 // For now we leave x64 off of the name for compatibility with other jobs
1214 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
1216 else if (scenario == 'formatting') {
1217 // we don't care about the configuration for the formatting job. It runs all configs
1218 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
1221 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1225 // These are cross builds
1226 assert os == 'Tizen'
1227 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1231 // These are cross builds
1232 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1235 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
1238 println("Unknown architecture: ${architecture}");
1243 return baseName + suffix
1246 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
1248 // The dev/unix_test_workflow branch is used for Jenkins CI testing. We generally do not need any non-PR
1249 // triggers in the branch, because that would use machine resources unnecessarily.
1250 if (branch == 'dev/unix_test_workflow') {
1254 // Limited hardware is restricted for non-PR triggers to certain branches.
1255 if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) {
1259 // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
1260 // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
1261 if (architecture == 'x86' && os == 'Ubuntu') {
1265 def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1269 case 'crossgen_comparison':
1270 if (isFlowJob && (configuration == 'Checked' || configuration == 'Release')) {
1271 if (os == 'Ubuntu' && architecture == 'arm') {
1272 // Not enough Linux/arm32 hardware for this.
1273 // addPeriodicTriggerHelper(job, '@daily')
1275 if (os == 'Ubuntu16.04' && architecture == 'arm64') {
1276 addPeriodicTriggerHelper(job, '@daily')
1281 case 'pmi_asm_diffs':
1282 // No non-PR triggers for now.
1286 switch (architecture) {
1289 if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
1290 addPeriodicTriggerHelper(job, '@daily')
1292 else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
1293 addGithubPushTriggerHelper(job)
1297 if (os == 'Windows_NT') {
1298 if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1299 // We would normally want a per-push trigger, but with limited hardware we can't keep up.
1300 // Do the builds daily.
1301 addPeriodicTriggerHelper(job, '@daily')
1305 // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1307 addPeriodicTriggerHelper(job, '@daily')
1312 if (os == 'Windows_NT') {
1313 if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) {
1314 // We would normally want a push trigger, but with limited hardware we can't keep up.
1315 // Do the builds daily.
1316 addPeriodicTriggerHelper(job, '@daily')
1320 assert os == 'Ubuntu'
1321 // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job.
1323 // Currently no push triggers, with limited arm Linux hardware.
1324 // TODO: If we have enough machine capacity, add some arm Linux push triggers.
1326 // Duplicated by AzDO
1327 // addPeriodicTriggerHelper(job, '@daily')
1332 addGithubPushTriggerHelper(job)
1335 println("Unknown architecture: ${architecture}");
1341 assert !(os in bidailyCrossList)
1342 // r2r gets a push trigger for checked/release
1343 if (configuration == 'Checked' || configuration == 'Release') {
1344 if (architecture == 'x64' && os != 'OSX10.12') {
1345 //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
1346 if (isFlowJob || os == 'Windows_NT') {
1347 addGithubPushTriggerHelper(job)
1349 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
1350 } else if (architecture == 'x64' && os == 'OSX10.12'){
1352 addPeriodicTriggerHelper(job, 'H H/12 * * *')
1355 // For x86, only add per-commit jobs for Windows
1356 else if (architecture == 'x86') {
1357 if (os == 'Windows_NT') {
1358 addGithubPushTriggerHelper(job)
1361 // arm r2r jobs should only run weekly.
1362 else if (architecture == 'arm') {
1364 // Linux arm32 done in AzDO
1365 if (os == 'Windows_NT') {
1366 addPeriodicTriggerHelper(job, '@weekly')
1370 // arm64 r2r jobs should only run weekly.
1371 else if (architecture == 'arm64') {
1373 addPeriodicTriggerHelper(job, '@weekly')
1378 case 'r2r_jitstress1':
1379 case 'r2r_jitstress2':
1380 case 'r2r_jitstress1_tiered':
1381 case 'r2r_jitstress2_tiered':
1382 case 'r2r_jitstressregs1':
1383 case 'r2r_jitstressregs2':
1384 case 'r2r_jitstressregs3':
1385 case 'r2r_jitstressregs4':
1386 case 'r2r_jitstressregs8':
1387 case 'r2r_jitstressregs0x10':
1388 case 'r2r_jitstressregs0x80':
1389 case 'r2r_jitstressregs0x1000':
1390 case 'r2r_jitminopts':
1391 case 'r2r_jitforcerelocs':
1392 case 'r2r_gcstress15':
1393 case 'r2r_no_tiered_compilation':
1394 assert !(os in bidailyCrossList)
1396 // GCStress=C is currently not supported on OS X
1397 if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
1401 if (configuration == 'Checked' || configuration == 'Release') {
1402 if (architecture == 'x64') {
1403 //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
1404 if (isFlowJob || os == 'Windows_NT') {
1405 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1408 // For x86, only add periodic jobs for Windows
1409 else if (architecture == 'x86') {
1410 if (os == 'Windows_NT') {
1411 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1414 else if (architecture == 'arm') {
1416 // Linux arm32 duplicated by AzDO
1417 if (os == 'Windows_NT') {
1418 addPeriodicTriggerHelper(job, '@weekly')
1422 else if (architecture == 'arm64') {
1424 addPeriodicTriggerHelper(job, '@weekly')
1430 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1431 assert configuration == 'Release'
1432 assert architecture == 'x64'
1433 addPeriodicTriggerHelper(job, '@daily')
1434 // TODO: Add once external email sending is available again
1435 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1438 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1439 assert configuration == 'Release'
1440 assert architecture == 'x64'
1441 addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
1442 // TODO: Add once external email sending is available again
1443 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1445 case 'standalone_gc':
1446 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1447 assert (configuration == 'Release' || configuration == 'Checked')
1448 // TODO: Add once external email sending is available again
1449 // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
1450 addPeriodicTriggerHelper(job, '@daily')
1452 case 'gc_reliability_framework':
1453 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1454 assert (configuration == 'Release' || configuration == 'Checked')
1455 // Only triggered by phrase.
1458 assert !(os in bidailyCrossList)
1459 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
1460 if (architecture == 'x64' && configuration == 'Release') {
1461 if (isFlowJob || os == 'Windows_NT') {
1462 addPeriodicTriggerHelper(job, '@daily')
1467 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
1468 assert configuration == 'Checked'
1469 assert (architecture == 'x64' || architecture == 'x86')
1470 addGithubPushTriggerHelper(job)
1473 assert (os == 'Windows_NT' || os == "Ubuntu")
1474 assert architecture == 'x64'
1475 addGithubPushTriggerHelper(job)
1477 case 'jitstressregs1':
1478 case 'jitstressregs2':
1479 case 'jitstressregs3':
1480 case 'jitstressregs4':
1481 case 'jitstressregs8':
1482 case 'jitstressregs0x10':
1483 case 'jitstressregs0x80':
1484 case 'jitstressregs0x1000':
1486 case 'tieredcompilation':
1487 case 'no_tiered_compilation':
1491 case 'jitstress1_tiered':
1492 case 'jitstress2_tiered':
1493 case 'jitstress2_jitstressregs1':
1494 case 'jitstress2_jitstressregs2':
1495 case 'jitstress2_jitstressregs3':
1496 case 'jitstress2_jitstressregs4':
1497 case 'jitstress2_jitstressregs8':
1498 case 'jitstress2_jitstressregs0x10':
1499 case 'jitstress2_jitstressregs0x80':
1500 case 'jitstress2_jitstressregs0x1000':
1501 case 'tailcallstress':
1504 case 'jitnox86hwintrinsic':
1505 case 'jitincompletehwintrinsic':
1506 case 'jitx86hwintrinsicnoavx':
1507 case 'jitx86hwintrinsicnoavx2':
1508 case 'jitx86hwintrinsicnosimd':
1509 case 'corefx_baseline':
1510 case 'corefx_minopts':
1511 case 'corefx_tieredcompilation':
1512 case 'corefx_jitstress1':
1513 case 'corefx_jitstress2':
1514 case 'corefx_jitstressregs1':
1515 case 'corefx_jitstressregs2':
1516 case 'corefx_jitstressregs3':
1517 case 'corefx_jitstressregs4':
1518 case 'corefx_jitstressregs8':
1519 case 'corefx_jitstressregs0x10':
1520 case 'corefx_jitstressregs0x80':
1521 case 'corefx_jitstressregs0x1000':
1523 if (os == 'CentOS7.1') {
1526 if (os in bidailyCrossList) {
1529 if ((os == 'Ubuntu') && (architecture == 'arm') && !isCoreFxScenario(scenario)) {
1530 // Linux arm32 duplicated by AzDO
1533 // ARM corefx testing uses non-flow jobs to provide the configuration-specific
1534 // build for the flow job. We don't need cron jobs for these. Note that the
1535 // Windows ARM jobs depend on a Windows "build only" job that exits the trigger
1536 // function very early, so only non-Windows gets here.
1537 if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) {
1540 if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
1543 if (jobRequiresLimitedHardware(architecture, os)) {
1544 if ((architecture == 'arm64') && (os == 'Ubuntu16.04')) {
1545 // These jobs are very fast on Linux/arm64 hardware, so run them daily.
1546 addPeriodicTriggerHelper(job, '@daily')
1548 else if (scenario == 'corefx_baseline') {
1549 addPeriodicTriggerHelper(job, '@daily')
1552 addPeriodicTriggerHelper(job, '@weekly')
1556 addPeriodicTriggerHelper(job, '@daily')
1561 if (os == 'CentOS7.1') {
1564 if (os in bidailyCrossList) {
1567 if ((os == 'Ubuntu') && (architecture == 'arm')) {
1568 // Linux arm32 duplicated by AzDO
1571 addPeriodicTriggerHelper(job, '@weekly')
1574 case 'gcstress0xc_zapdisable':
1575 case 'gcstress0xc_zapdisable_jitstress2':
1576 case 'gcstress0xc_zapdisable_heapverify1':
1577 case 'gcstress0xc_jitstress1':
1578 case 'gcstress0xc_jitstress2':
1579 case 'gcstress0xc_minopts_heapverify1':
1580 if (os == 'OSX10.12') {
1581 // GCStress=C is currently not supported on OS X
1584 if (os == 'CentOS7.1') {
1587 if (os in bidailyCrossList) {
1590 if ((os == 'Ubuntu') && (architecture == 'arm')) {
1591 // Linux arm32 duplicated by AzDO
1594 addPeriodicTriggerHelper(job, '@weekly')
1598 // Testing on other operating systems TBD
1599 assert (os == 'Windows_NT' || os == 'Ubuntu')
1600 if (architecture == 'x64' || architecture == 'x86') {
1601 if (configuration == 'Checked') {
1602 addPeriodicTriggerHelper(job, '@daily')
1608 println("Unknown scenario: ${scenario}");
1615 // **************************
1616 // Define the basic inner loop builds for PR and commit. This is basically just the set
1617 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked. In addition, the windows
1618 // builds will do a couple extra steps.
1619 // **************************
1621 // Adds a trigger for the PR build if one is needed. If isFlowJob is true, then this is the
1622 // flow job that rolls up the build and test for non-windows OS's. // If the job is a windows build only job,
1623 // it's just used for internal builds
1624 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
1625 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
1626 def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop")
1628 if (isWindowsBuildOnlyJob) {
1632 def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
1633 // Non pull request builds.
1635 addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
1660 // Pull request builds. Generally these fall into two categories: default triggers and on-demand triggers
1661 // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
1663 def contextString = ""
1664 def triggerString = ""
1665 def needsTrigger = true
1666 def isDefaultTrigger = false
1667 def isArm64PrivateJob = false
1668 def scenarioString = ""
1670 // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep
1671 // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for
1672 // some jobs where the Debug configuration just does builds, no tests.
1674 // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string.
1675 // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>"
1676 // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either
1677 // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up.
1679 if (architecture == 'armem') {
1680 assert os == 'Tizen'
1681 architecture = 'armel'
1684 switch (architecture) {
1688 contextString = "${os} ${architecture} Cross ${configuration}"
1689 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1691 if (scenario == 'innerloop') {
1692 contextString += " Innerloop"
1693 triggerString += "\\W+Innerloop"
1696 contextString += " ${scenario}"
1697 triggerString += "\\W+${scenario}"
1700 if (scenario == 'pmi_asm_diffs') {
1701 // Don't add the "Build and Test" part
1703 else if (configuration == 'Debug') {
1704 contextString += " Build"
1705 triggerString += "\\W+Build"
1708 contextString += " Build and Test"
1709 triggerString += "\\W+Build and Test"
1712 triggerString += ".*"
1716 scenarioString = getScenarioDisplayString(scenario)
1717 contextString = "${os} ${architecture} ${configuration} ${scenarioString}"
1718 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}"
1722 triggerString += "\\W+Build and Test.*"
1725 case 'corefx_innerloop': // maintain this asymmetry
1726 triggerString += "\\W+CoreFX Tests.*"
1730 triggerString += "\\W+${scenario}.*"
1734 triggerString += ".*"
1738 // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for
1739 // non-flow jobs that are only used as part of flow jobs.
1741 switch (architecture) {
1742 case 'x64': // editor brace matching: {
1743 if (scenario == 'formatting') {
1744 assert configuration == 'Checked'
1745 if (os == 'Windows_NT' || os == 'Ubuntu') {
1746 isDefaultTrigger = true
1747 contextString = "${os} ${architecture} Formatting"
1752 if (scenario == 'pmi_asm_diffs') {
1753 // Everything is already set.
1754 // No default triggers.
1759 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1762 if (scenario == 'innerloop') {
1764 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1765 isDefaultTrigger = true
1775 assert scenario != 'innerloop'
1776 contextString = "${os} ${architecture} ${configuration} Build"
1777 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*"
1781 if (scenario == 'illink') {
1784 else if (scenario == 'corefx_innerloop') {
1785 if (configuration == 'Checked') {
1786 isDefaultTrigger = true
1794 // Triggers on the non-flow jobs aren't necessary here
1795 // Corefx testing uses non-flow jobs.
1796 if (!isFlowJob && !isCoreFxScenario(scenario)) {
1797 needsTrigger = false
1802 isDefaultTrigger = true
1805 case 'no_tiered_compilation_innerloop':
1806 if (os == 'Ubuntu') {
1807 isDefaultTrigger = true
1819 // CentOS uses checked for default PR tests while debug is build only
1820 if (configuration == 'Debug') {
1821 isDefaultTrigger = true
1822 contextString = "${os} ${architecture} ${configuration} Innerloop Build"
1826 // Make sure this is a flow job to get build and test.
1828 needsTrigger = false
1832 if (configuration == 'Checked') {
1833 assert job.name.contains("flow")
1834 isDefaultTrigger = true
1835 contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test"
1840 // Make sure this is a flow job to get build and test.
1842 needsTrigger = false
1855 case 'no_tiered_compilation_innerloop':
1856 isDefaultTrigger = true
1859 case 'corefx_innerloop':
1860 if (configuration == 'Checked' || configuration == 'Release') {
1861 isDefaultTrigger = true
1871 println("Unknown os: ${os}");
1878 // editor brace matching: }
1880 case 'armel': // editor brace matching: {
1883 azureVMAgentPostBuildAction {
1884 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1891 if (scenario == 'innerloop') {
1892 if (configuration == 'Checked') {
1893 isDefaultTrigger = true
1900 // editor brace matching: }
1903 case 'arm64': // editor brace matching: {
1909 // Triggers on the non-flow jobs aren't necessary
1911 needsTrigger = false
1914 if (os == 'Ubuntu' && architecture == 'arm') {
1917 case 'no_tiered_compilation_innerloop':
1918 if (configuration == 'Checked') {
1919 isDefaultTrigger = true
1922 case 'crossgen_comparison':
1923 if (configuration == 'Checked' || configuration == 'Release') {
1924 isDefaultTrigger = true
1932 assert isArmWindowsScenario(scenario)
1934 // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for
1935 // the non-flow Build job. All others need a trigger on the flow job.
1936 def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug'))
1937 if (isFlowJob != needsFlowJobTrigger) {
1938 needsTrigger = false
1944 if (configuration == 'Checked') {
1945 isDefaultTrigger = true
1946 isArm64PrivateJob = true
1950 isArm64PrivateJob = true
1955 println("NYI os: ${os}");
1961 // editor brace matching: }
1962 case 'x86': // editor brace matching: {
1963 assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop))
1964 if (os == 'Ubuntu') {
1965 // Triggers on the non-flow jobs aren't necessary here
1967 needsTrigger = false
1971 // on-demand only for ubuntu x86
1972 contextString = "${os} ${architecture} ${configuration} Build"
1973 triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*"
1978 case 'no_tiered_compilation_innerloop':
1979 isDefaultTrigger = true
1986 // editor brace matching: }
1989 println("Unknown architecture: ${architecture}");
1995 if (isArm64PrivateJob) {
1996 // ignore isDefaultTrigger to disable Jenkins by default
1998 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
2001 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
2005 // ignore isDefaultTrigger to disable Jenkins by default
2007 Utilities.addGithubPRTriggerForBranch(job, branch, contextString)
2010 Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString)
2016 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
2017 def buildCommands = []
2018 def osGroup = getOSGroup(os)
2019 def lowerConfiguration = configuration.toLowerCase()
2021 // Which set of tests to build? Innerloop tests build Pri-0.
2022 // Currently, we only generate asm diffs on Pri-0 tests, if we generate asm diffs on tests at all.
2023 // CoreFX testing skipts building tests altogether (done below).
2024 // All other scenarios build Pri-1 tests.
2026 if (isInnerloopTestScenario(scenario)) {
2030 def doCoreFxTesting = isCoreFxScenario(scenario)
2032 def buildCoreclrTests = true
2033 if (doCoreFxTesting || (scenario == 'pmi_asm_diffs')) {
2034 // These scenarios don't need the coreclr tests build.
2035 buildCoreclrTests = false
2038 // Calculate the build steps, archival, and xunit results
2040 case 'Windows_NT': // editor brace matching: {
2041 switch (architecture) {
2044 def arch = architecture
2047 if (scenario == 'formatting') {
2048 buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
2049 Utilities.addArchival(newJob, "format.patch", "", true, false)
2053 if (scenario == 'illink') {
2054 buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
2057 // If it is a release build for Windows, ensure PGO is used, else fail the build.
2058 if ((lowerConfiguration == 'release') &&
2059 (scenario in Constants.basicScenarios)) {
2061 buildOpts += ' -enforcepgo'
2064 if (buildCoreclrTests) {
2065 buildOpts += " -priority=${priority}"
2067 buildOpts += ' skiptests';
2070 // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
2071 // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
2072 // 35 characters long.
2074 buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
2076 if (scenario == 'pmi_asm_diffs') {
2077 // Now, generate the layout. We don't have any tests, but we need to restore the packages before calling runtest.cmd.
2078 // Call build-test.cmd to do this. It will do a little more than we need, but that's ok.
2079 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} skipmanaged skipnative"
2080 buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${arch} GenerateLayoutOnly"
2082 // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2083 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-pmi-diffs.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration}"
2086 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('_\\pmi\\asm', '.\\dasm.${os}.${architecture}.${configuration}.zip')\"";
2089 Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip")
2094 def runtestArguments = ''
2095 def testOpts = 'collectdumps'
2097 if (isR2RScenario(scenario)) {
2099 // If this is a ReadyToRun scenario, pass 'crossgen'
2100 // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
2101 // to cause the tests to be crossgen'ed.
2103 testOpts += ' crossgen runcrossgentests'
2105 else if (scenario == 'jitdiff') {
2106 testOpts += ' jitdisasm crossgen'
2108 else if (scenario == 'ilrt') {
2109 testOpts += ' ilasmroundtrip'
2111 else if (isLongGc(scenario)) {
2112 testOpts += " ${scenario} sequential"
2114 else if (scenario == 'standalone_gc') {
2115 testOpts += ' gcname clrgc.dll'
2117 else if (scenario == 'illink') {
2118 testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
2121 // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
2122 // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
2123 // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
2124 // which is read by the xunit harness.)
2125 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
2127 def timeout = 1800000
2128 testOpts += " timeout ${timeout}"
2131 // If we are running a stress mode, we should write out the set of key
2132 // value env pairs to a file at this point and then we'll pass that to runtest.cmd
2134 def envScriptPath = ''
2135 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2136 def buildCommandsStr = ''
2137 envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2138 buildCommandsStr += envScriptCreate(os, envScriptPath)
2140 if (isJitStressScenario(scenario)) {
2141 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2143 else if (isR2RStressScenario(scenario)) {
2144 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
2147 envScriptFinalize(os, envScriptPath)
2149 // Note that buildCommands is an array of individually executed commands; we want all the commands used to
2150 // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
2151 // into a single script.
2152 buildCommands += buildCommandsStr
2154 if (envScriptPath != '') {
2155 testOpts += " TestEnv ${envScriptPath}"
2158 runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
2160 if (doCoreFxTesting) {
2161 if (scenario == 'corefx_innerloop') {
2162 // Create CORE_ROOT and testhost
2163 buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly"
2164 buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll"
2166 // Archive and process (only) the test results
2167 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2168 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2171 def workspaceRelativeFxRoot = "_/fx"
2172 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2173 def fxBranch = getFxBranch(branch)
2174 def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
2176 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -exclusion_rsp_file ${exclusionRspPath}"
2178 // Archive and process (only) the test results
2179 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2180 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml")
2182 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
2183 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
2184 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/testhost/**", "", true, false)
2187 else if (isGcReliabilityFramework(scenario)) {
2188 buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
2189 buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
2192 def buildCommandsStr = "call tests\\runtest.cmd ${runtestArguments}\r\n"
2194 // If we ran the tests, collect the test logs collected by xunit. We want to do this even if the tests fail, so we
2195 // must do it in the same batch file as the test run.
2197 buildCommandsStr += "echo on\r\n" // Show the following commands in the log. "echo" doesn't alter the errorlevel.
2198 buildCommandsStr += "set saved_errorlevel=%errorlevel%\r\n"
2199 buildCommandsStr += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\"\r\n";
2200 buildCommandsStr += "exit /b %saved_errorlevel%\r\n"
2202 def doNotFailIfNothingArchived = true
2203 def archiveOnlyIfSuccessful = false
2204 Utilities.addArchival(newJob, "bin/tests/testReports.zip", "", doNotFailIfNothingArchived, archiveOnlyIfSuccessful)
2206 buildCommands += buildCommandsStr
2208 } // end if (!isBuildOnly)
2210 if (!doCoreFxTesting) {
2211 // Run the rest of the build
2212 // Build the mscorlib for the other OS's
2213 buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
2214 buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
2216 if (arch == 'x64') {
2217 buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
2220 if (!isJitStressScenario(scenario)) {
2221 // Zip up the tests directory so that we don't use so much space/time copying
2222 // 10s of thousands of files around.
2223 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2225 // For Windows, pull full test results and test drops for x86/x64.
2226 // No need to pull for stress mode scenarios (downstream builds use the default scenario)
2227 Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2230 if (scenario == 'jitdiff') {
2231 // retrieve jit-dasm output for base commit, and run jit-diff
2233 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
2234 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
2239 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
2243 // Archive the logs, even if the build failed (which is when they are most interesting).
2244 Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2248 assert isArmWindowsScenario(scenario)
2252 if (buildCoreclrTests) {
2253 buildOpts += " -priority=${priority}"
2255 buildOpts += ' skiptests'
2258 // This is now a build only job. Do not run tests. Use the flow job.
2259 buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
2261 if (doCoreFxTesting) {
2264 // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files.
2265 def envScriptPath = ''
2266 def buildCommandsStr = ''
2267 envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
2268 buildCommandsStr += envScriptCreate(os, envScriptPath)
2269 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
2270 envScriptFinalize(os, envScriptPath)
2271 buildCommands += buildCommandsStr
2273 def workspaceRelativeFxRootLinux = "_/fx"
2274 def workspaceRelativeFxRootWin = "_\\fx"
2275 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
2276 def fxBranch = getFxBranch(branch)
2278 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests"
2280 // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree.
2281 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\"";
2282 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\artifacts\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\"";
2284 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2285 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2287 // Zip up the tests directory so that we don't use so much space/time copying
2288 // 10s of thousands of files around.
2289 buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
2292 Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
2295 // Archive the logs, even if the build failed (which is when they are most interesting).
2296 Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2299 println("Unknown architecture: ${architecture}");
2304 // end case 'Windows_NT'; editor brace matching: }
2313 case 'Fedora24': // editor brace matching: {
2314 switch (architecture) {
2316 if (os == 'Ubuntu') {
2317 // build and PAL test
2318 def dockerImage = getDockerImageName(architecture, os, true)
2319 buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
2320 dockerImage = getDockerImageName(architecture, os, false)
2321 buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
2322 Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2323 Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2328 if (scenario == 'formatting') {
2329 buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
2330 Utilities.addArchival(newJob, "format.patch", "", true, false)
2334 if (scenario == 'pmi_asm_diffs') {
2335 buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests skipbuildpackages"
2336 buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatelayoutonly"
2338 // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2339 buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration}"
2342 buildCommands += "tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm"
2343 Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.tgz")
2347 if (scenario == 'illink') {
2348 assert(os == 'Ubuntu')
2349 buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
2352 if (!doCoreFxTesting) {
2353 // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
2354 // only on supported OS platforms.
2355 def bootstrapRid = Utilities.getBoostrapPublishRid(os)
2356 def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
2358 buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}"
2360 def testBuildOpts = ""
2361 if (priority == '1') {
2362 testBuildOpts = "priority1"
2365 buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} ${testBuildOpts}"
2366 buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
2368 // Archive the bin/tests folder for *_tst jobs
2369 def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration)
2370 buildCommands += "tar -czf ${testArtifactsTgzFileName} bin/tests/${osGroup}.${architecture}.${configuration}"
2371 Utilities.addArchival(newJob, "${testArtifactsTgzFileName}", "")
2373 Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
2376 if (scenario == 'corefx_innerloop') {
2377 assert os == 'Ubuntu' || 'OSX10.12'
2378 assert architecture == 'x64'
2380 buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests"
2381 buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly"
2382 buildCommands += "./tests/runtest.sh ${lowerConfiguration} --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}"
2384 // Archive and process (only) the test results
2385 Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2386 Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml")
2389 // Corefx stress testing
2390 assert os == 'Ubuntu'
2391 assert architecture == 'x64'
2392 assert lowerConfiguration == 'checked'
2393 assert isJitStressScenario(scenario)
2396 buildCommands += "./build.sh ${lowerConfiguration} ${architecture}"
2398 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2400 def envScriptCmds = envScriptCreate(os, scriptFileName)
2401 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2402 envScriptCmds += envScriptFinalize(os, scriptFileName)
2403 buildCommands += envScriptCmds
2405 // Build and text corefx
2406 def workspaceRelativeFxRoot = "_/fx"
2407 def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
2408 def fxBranch = getFxBranch(branch)
2409 def exclusionRspPath = "\$WORKSPACE/tests/scripts/run-corefx-tests-exclusions.txt"
2411 buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -exclusion_rsp_file ${exclusionRspPath}"
2413 // Archive and process (only) the test results
2414 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2415 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml")
2419 // Archive the logs, even if the build failed (which is when they are most interesting).
2420 Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2423 // Emulator cross builds for ARM runs on Tizen currently
2424 assert os == 'Tizen'
2426 def arm_abi = "armel"
2427 def linuxCodeName = "tizen"
2429 // Unzip the Windows test binaries first. Exit with 0
2430 buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2432 // Unpack the corefx binaries
2433 buildCommands += "mkdir ./bin/CoreFxBinDir"
2434 buildCommands += "tar -xf ./artifacts/bin/build.tar.gz -C ./bin/CoreFxBinDir"
2436 // Call the ARM CI script to cross build and test using docker
2437 buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2440 --linuxCodeName=${linuxCodeName} \\
2441 --buildConfig=${lowerConfiguration} \\
2442 --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2443 --coreFxBinDir=./bin/CoreFxBinDir \\
2444 --testDirFile=./tests/testsRunningInsideARM.txt"""
2446 // Basic archiving of the build, no pal tests
2447 Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
2451 // Non-Windows ARM cross builds on hardware run on Ubuntu only
2452 assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
2454 // Add some useful information to the log file. Ignore return codes.
2455 buildCommands += "uname -a || true"
2457 // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct
2458 // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host).
2459 // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product
2460 // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we
2461 // ZIP up the generated CoreFX runtime and tests.
2463 def dockerImage = getDockerImageName(architecture, os, true)
2464 def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${dockerImage} "
2466 buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross"
2468 if (doCoreFxTesting) {
2469 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2471 def envScriptCmds = envScriptCreate(os, scriptFileName)
2472 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2473 envScriptCmds += envScriptFinalize(os, scriptFileName)
2474 buildCommands += envScriptCmds
2476 // Build and text corefx
2477 def workspaceRelativeFxRootLinux = "_/fx"
2478 def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}"
2479 def fxBranch = getFxBranch(branch)
2481 buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests"
2483 // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors.
2484 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture}"
2485 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/artifacts/bin/tests"
2487 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip")
2488 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
2490 else if (isCrossGenComparisonScenario(scenario)) {
2491 buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2493 def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
2494 def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
2495 def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
2496 def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
2497 def workspaceRelativeResultsDir = "_"
2498 def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
2499 def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
2500 def crossArch = "x64"
2501 def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen"
2502 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
2504 buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2505 buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2506 buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}"
2508 buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
2509 Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
2511 else if (scenario == 'pmi_asm_diffs') {
2512 buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
2514 // Pass `--skip_diffs` -- the actual diffs will be done on an arm machine in the test job. This is the build job.
2515 // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables.
2516 buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} --skip_diffs"
2518 // Archive what we created.
2519 buildCommands += "tar -czf product.${os}.${architecture}.${lowerConfiguration}.tgz ./bin/Product/Linux.${architecture}.${configuration}"
2520 buildCommands += "tar -czf product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz ./_/pmi/base/bin/Product/Linux.${architecture}.${configuration}"
2521 buildCommands += "tar -czf coreroot.${os}.${architecture}.${lowerConfiguration}.tgz ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2522 buildCommands += "tar -czf coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz ./_/pmi/base/bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root"
2524 // Archive the built artifacts
2525 Utilities.addArchival(newJob, "product.${os}.${architecture}.${lowerConfiguration}.tgz,product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz,coreroot.${os}.${architecture}.${lowerConfiguration}.tgz,coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz")
2528 // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
2530 def testBuildOpts = ""
2531 if (priority == '1') {
2532 testBuildOpts = "priority1"
2535 buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
2537 // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
2538 def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration)
2539 buildCommands += "tar -czf ${testArtifactsTgzFileName} bin/tests/${osGroup}.${architecture}.${configuration}"
2541 Utilities.addArchival(newJob, "${testArtifactsTgzFileName}", "")
2544 // Archive the logs, even if the build failed (which is when they are most interesting).
2545 Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
2547 // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
2548 // the cleanup task in Jenkins can't remove.
2551 azureVMAgentPostBuildAction {
2552 agentPostBuildAction('Delete agent after build execution (when idle).')
2558 println("Unknown architecture: ${architecture}");
2563 // editor brace matching: }
2565 println("Unknown os: ${os}");
2570 return buildCommands
2573 // Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only.
2574 // Returns true if the job should be generated.
2575 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
2577 def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
2579 // Innerloop jobs (except corefx_innerloop) are no longer created in Jenkins
2580 // The only exception is windows arm(64)
2581 if (isInnerloopTestScenario(scenario) && isPR && !windowsArmJob) {
2582 assert scenario != 'corefx_innerloop'
2587 if (isInnerloopTestScenario(scenario)) {
2591 if (scenario == 'corefx_innerloop') {
2596 // Tizen is only supported for armem architecture
2597 if (os == 'Tizen' && architecture != 'armem') {
2601 // Filter based on architecture.
2603 switch (architecture) {
2605 if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2610 if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
2615 if (os != 'Tizen') {
2620 if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
2625 // Everything implemented
2628 println("Unknown architecture: ${architecture}")
2633 // Which (Windows) build only jobs are required?
2635 def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
2638 switch (architecture) {
2641 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2642 if (!isCoreFxScenario(scenario)) {
2648 if (!isNormalOrInnerloop) {
2657 // Filter based on scenario.
2659 if (isJitStressScenario(scenario)) {
2660 if (configuration != 'Checked') {
2664 def isEnabledOS = (os == 'Windows_NT') ||
2665 (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
2666 (os == 'Ubuntu' && architecture == 'arm') ||
2667 (os == 'Ubuntu16.04' && architecture == 'arm64')
2672 switch (architecture) {
2677 // x86 ubuntu: no stress modes
2678 if (os == 'Ubuntu') {
2685 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
2686 // No "regular" Windows arm corefx jobs, e.g.
2687 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
2688 // the Ubuntu arm "regular" jobs don't run tests anyway).
2689 if (os == 'Windows_NT') {
2690 if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
2695 if (!isCoreFxScenario(scenario)) {
2702 // armem: no stress jobs for ARM emulator.
2706 else if (isR2RScenario(scenario)) {
2707 if (os != 'Windows_NT') {
2711 if (isR2RBaselineScenario(scenario)) {
2712 // no need for Debug scenario; Checked is sufficient
2713 if (configuration != 'Checked' && configuration != 'Release') {
2717 else if (isR2RStressScenario(scenario)) {
2718 // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
2719 if (configuration != 'Checked') {
2724 switch (architecture) {
2727 // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs.
2734 else if (isCrossGenComparisonScenario(scenario)) {
2735 return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
2741 // The ilrt build isn't necessary except for Windows_NT2003. Non-Windows NT uses
2742 // the default scenario build
2743 if (os != 'Windows_NT') {
2747 if (architecture != 'x64') {
2751 if (configuration != 'Release') {
2756 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2759 if (architecture != 'x64') {
2762 if (configuration != 'Checked') {
2768 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2771 if (architecture != 'x64') {
2774 if (configuration != 'Release') {
2778 case 'gc_reliability_framework':
2779 case 'standalone_gc':
2780 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2784 if (architecture != 'x64') {
2788 if (configuration != 'Release' && configuration != 'Checked') {
2792 // We only run Windows and Ubuntu x64 Checked for formatting right now
2794 if (os != 'Windows_NT' && os != 'Ubuntu') {
2797 if (architecture != 'x64') {
2800 if (configuration != 'Checked') {
2805 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2808 if (architecture != 'x64' && architecture != 'x86') {
2816 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) {
2820 case 'corefx_innerloop':
2821 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2824 if (architecture != 'x64') {
2828 case 'pmi_asm_diffs':
2829 if (configuration != 'Checked') {
2832 if (architecture == 'armem') {
2835 // Currently, we don't support pmi_asm_diffs for Windows arm/arm64. We don't have a dotnet CLI available to
2836 // build jitutils. The jobs are not in validArmWindowsScenarios.
2837 if ((os == 'Windows_NT') && (architecture == 'arm' || architecture == 'arm64')) {
2840 // Currently, no support for Linux x86.
2841 if ((os != 'Windows_NT') && (architecture == 'x86')) {
2846 println("Unknown scenario: ${scenario}")
2852 // The job was not filtered out, so we should generate it!
2856 Constants.allScenarios.each { scenario ->
2857 [true, false].each { isPR ->
2858 Constants.architectureList.each { architecture ->
2859 Constants.configurationList.each { configuration ->
2860 Constants.osList.each { os ->
2861 // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2862 // and reset the os to Windows_NT
2863 def isBuildOnly = false
2864 if (os == 'Windows_NT_BuildOnly') {
2869 if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) {
2874 def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2875 def folderName = getJobFolder(scenario)
2877 // Create the new job
2878 def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2880 addToViews(newJob, false, isPR, architecture, os, configuration, scenario) // isFlowJob == false
2882 setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false
2884 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2885 addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2886 setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
2888 // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem.
2889 // We don't use a flow job for this, but we do depend on there being existing builds with these
2890 // artifacts produced.
2891 if ((architecture == 'armem') && (os == 'Tizen')) {
2892 // Define the Windows Tests and Corefx build job names
2893 def lowerConfiguration = configuration.toLowerCase()
2894 def WindowsTestsName = projectFolder + '/' +
2895 Utilities.getFullJobName(project,
2896 getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true),
2898 def fxBranch = getFxBranch(branch)
2899 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2900 Utilities.getFolderName(fxBranch)
2902 def arm_abi = 'armel'
2903 def corefx_os = 'tizen'
2905 // Let's use release CoreFX to test checked CoreCLR,
2906 // because we do not generate checked CoreFX in CoreFX CI yet.
2907 def corefx_lowerConfiguration = lowerConfiguration
2908 if (lowerConfiguration == 'checked') {
2909 corefx_lowerConfiguration = 'release'
2912 // Copy the Windows test binaries and the Corefx build binaries
2915 copyArtifacts(WindowsTestsName) {
2916 includePatterns('bin/tests/tests.zip')
2918 latestSuccessful(true)
2921 copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2922 includePatterns('artifacts/bin/build.tar.gz')
2924 latestSuccessful(true)
2931 def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2935 if (os == 'Windows_NT') {
2936 buildCommands.each { buildCommand ->
2937 batchFile(buildCommand)
2941 buildCommands.each { buildCommand ->
2954 // Create a Windows ARM/ARM64 test job that will be used by a flow job.
2955 // Returns the newly created job.
2956 def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
2958 def osGroup = getOSGroup(os)
2959 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2961 def jobFolder = getJobFolder(scenario)
2962 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
2964 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2968 // Set up the copies
2970 // Coreclr build we are trying to test
2972 // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
2974 copyArtifacts(inputCoreCLRBuildName) {
2975 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2977 buildNumber('${CORECLR_BUILD}')
2981 if (isCoreFxScenario(scenario)) {
2983 // Only arm/arm64 supported for corefx testing now.
2984 assert architecture == 'arm' || architecture == 'arm64'
2986 // Unzip CoreFx runtime
2987 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"")
2989 // Unzip CoreFx tests.
2990 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\artifacts\\bin\\tests')\"")
2992 // Add the script to run the corefx tests
2993 def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
2994 def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\tests"
2995 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
2996 def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
2997 batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture} ${exclusionRspPath}")
2999 } else { // !isCoreFxScenario(scenario)
3002 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"")
3004 def buildCommands = ""
3006 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
3007 def addEnvVariable = { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
3008 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
3010 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
3011 addCommand("SETLOCAL ENABLEEXTENSIONS")
3014 addEnvVariable("CORE_ROOT", coreRootLocation)
3015 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
3016 addEnvVariable("COMPlus_ContinueOnAssert", "0")
3018 // If we are running a stress mode, we'll set those variables as well
3019 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
3020 def stressValues = null
3021 if (isJitStressScenario(scenario)) {
3022 stressValues = Constants.jitStressModeScenarios[scenario]
3025 stressValues = Constants.r2rStressScenarios[scenario]
3028 stressValues.each { key, value ->
3029 addEnvVariable(key, value)
3033 if (isR2RScenario(scenario)) {
3034 // Crossgen the framework assemblies.
3035 buildCommands += """
3036 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
3037 @goto skip_PrecompileAssembly
3040 @REM Skip mscorlib since it is already precompiled.
3041 @if /I "%3" == "mscorlib.dll" exit /b 0
3042 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
3044 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
3045 @if "%errorlevel%" == "-2146230517" (
3046 echo %2 is not a managed assembly.
3047 ) else if "%errorlevel%" == "-2146234344" (
3048 echo %2 is not a managed assembly.
3049 ) else if %errorlevel% neq 0 (
3050 echo Unable to precompile %2
3056 :skip_PrecompileAssembly
3059 // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
3060 // crossgen on tests before running them.
3061 addEnvVariable("RunCrossGen", "true")
3062 } // isR2RScenario(scenario)
3065 // Do not run generate layout. It will delete the correct CORE_ROOT, and we do not have a correct product
3066 // dir to copy from.
3067 def runtestCommand = "call %WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout"
3069 addCommand("${runtestCommand}")
3070 addCommand("echo on") // Show the following commands in the log. "echo" doesn't alter the errorlevel.
3071 addCommand("set saved_errorlevel=%errorlevel%")
3073 // Collect the test logs collected by xunit. Ignore errors here. We want to collect these even if the run
3074 // failed for some reason, so it needs to be in this batch file.
3076 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\"");
3078 // Use the runtest.cmd errorlevel as the script errorlevel.
3079 addCommand("exit /b %saved_errorlevel%")
3081 batchFile(buildCommands)
3082 } // non-corefx testing
3086 if (!isCoreFxScenario(scenario)) {
3087 def doNotFailIfNothingArchived = true
3088 def archiveOnlyIfSuccessful = false
3089 Utilities.addArchival(newJob, "bin/tests/testReports.zip", "", doNotFailIfNothingArchived, archiveOnlyIfSuccessful)
3091 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
3097 // Create a test job not covered by the "Windows ARM" case that will be used by a flow job.
3098 // E.g., non-Windows tests.
3099 // Returns the newly created job.
3100 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3102 def lowerConfiguration = configuration.toLowerCase()
3104 def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
3105 def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
3106 def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
3108 def doCoreFxTesting = isCoreFxScenario(scenario)
3109 def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs')
3111 def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
3113 def osGroup = getOSGroup(os)
3114 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3117 def useServerGC = false
3119 // Enable Server GC for Ubuntu PR builds
3120 // REVIEW: why? Does this apply to all architectures? Why only PR?
3121 if (os == 'Ubuntu' && isPR) {
3122 testOpts += ' --useServerGC'
3126 if (isR2RScenario(scenario)) {
3128 testOpts += ' --crossgen --runcrossgentests'
3130 if (scenario == 'r2r_jitstress1') {
3131 testOpts += ' --jitstress=1'
3133 else if (scenario == 'r2r_jitstress2') {
3134 testOpts += ' --jitstress=2'
3136 else if (scenario == 'r2r_jitstress1_tiered') {
3137 testOpts += ' --jitstress=1'
3139 else if (scenario == 'r2r_jitstress2_tiered') {
3140 testOpts += ' --jitstress=2'
3142 else if (scenario == 'r2r_jitstressregs1') {
3143 testOpts += ' --jitstressregs=1'
3145 else if (scenario == 'r2r_jitstressregs2') {
3146 testOpts += ' --jitstressregs=2'
3148 else if (scenario == 'r2r_jitstressregs3') {
3149 testOpts += ' --jitstressregs=3'
3151 else if (scenario == 'r2r_jitstressregs4') {
3152 testOpts += ' --jitstressregs=4'
3154 else if (scenario == 'r2r_jitstressregs8') {
3155 testOpts += ' --jitstressregs=8'
3157 else if (scenario == 'r2r_jitstressregs0x10') {
3158 testOpts += ' --jitstressregs=0x10'
3160 else if (scenario == 'r2r_jitstressregs0x80') {
3161 testOpts += ' --jitstressregs=0x80'
3163 else if (scenario == 'r2r_jitstressregs0x1000') {
3164 testOpts += ' --jitstressregs=0x1000'
3166 else if (scenario == 'r2r_jitminopts') {
3167 testOpts += ' --jitminopts'
3169 else if (scenario == 'r2r_jitforcerelocs') {
3170 testOpts += ' --jitforcerelocs'
3172 else if (scenario == 'r2r_gcstress15') {
3173 testOpts += ' --gcstresslevel=0xF'
3176 else if (scenario == 'jitdiff') {
3177 testOpts += ' --jitdisasm --crossgen'
3179 else if (scenario == 'illink') {
3180 testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
3182 else if (isLongGc(scenario)) {
3183 // Long GC tests behave very poorly when they are not
3184 // the only test running (many of them allocate until OOM).
3185 testOpts += ' --sequential'
3187 // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
3188 // for running long GC and GCSimulator tests, respectively. We don't use them
3189 // here because using a playlist file produces much more readable output on the CI machines
3190 // and reduces running time.
3192 // The Long GC playlist contains all of the tests that are
3193 // going to be run. The GCSimulator playlist contains all of
3194 // the GC simulator tests.
3195 if (scenario == 'longgc') {
3196 testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
3198 else if (scenario == 'gcsimulator') {
3199 testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
3202 else if (isGcReliabilityFramework(scenario)) {
3203 testOpts += ' --build-overlay-only'
3205 else if (scenario == 'standalone_gc') {
3206 if (osGroup == 'OSX') {
3207 testOpts += ' --gcname=libclrgc.dylib'
3209 else if (osGroup == 'Linux') {
3210 testOpts += ' --gcname=libclrgc.so'
3213 println("Unexpected OS group: ${osGroup} for os ${os}")
3218 def jobFolder = getJobFolder(scenario)
3219 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3221 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3225 // Set up the copies
3227 // Coreclr build we are trying to test
3229 // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
3230 // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
3232 if (!isUbuntuArm64Job) {
3233 copyArtifacts(inputCoreCLRBuildName) {
3234 excludePatterns('**/testResults.xml', '**/*.ni.dll')
3236 buildNumber('${CORECLR_BUILD}')
3241 if (isUbuntuArmJob) {
3242 // Add some useful information to the log file. Ignore return codes.
3243 shell("uname -a || true")
3246 if (isUbuntuArm64Job) {
3247 // Copy the required artifacts directly, using wget, e.g.:
3249 // https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
3250 // https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
3252 // parameterized as:
3254 // https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
3255 // https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
3257 // CoreFX example artifact URLs:
3259 // https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
3260 // https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
3262 // Note that the source might be in a "jitstress" folder.
3264 // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
3266 // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
3268 shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
3270 def mungedProjectName = Utilities.getFolderName(project)
3271 def mungedBranchName = Utilities.getFolderName(branch)
3273 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3274 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3275 if (isPmiAsmDiffsScenario || doCoreFxTesting || doCrossGenComparison) {
3276 // These depend on unique builds for each scenario
3277 inputCoreCLRBuildScenario = scenario
3279 def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
3280 def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
3282 // Need to add the sub-folder if necessary.
3283 def inputJobPath = "job/${inputJobName}"
3284 def folderName = getJobFolder(inputCoreCLRBuildScenario)
3285 if (folderName != '') {
3286 inputJobPath = "job/${folderName}/job/${inputJobName}"
3289 def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
3291 if (isPmiAsmDiffsScenario) {
3292 def workspaceRelativeRootLinux = "_/pmi"
3293 shell("mkdir -p ${workspaceRelativeRootLinux}")
3294 shell("wget --progress=dot:giga ${inputUrlRoot}/product.${os}.${architecture}.${lowerConfiguration}.tgz")
3295 shell("wget --progress=dot:giga ${inputUrlRoot}/product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz")
3296 shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.${os}.${architecture}.${lowerConfiguration}.tgz")
3297 shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz")
3299 else if (doCoreFxTesting) {
3300 shell("mkdir -p ${workspaceRelativeFxRootLinux}")
3301 shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
3302 shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
3305 def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration)
3306 shell("wget --progress=dot:giga ${inputUrlRoot}/${testArtifactsTgzFileName}")
3310 if (architecture == 'x86') {
3311 shell("mkdir ./bin/CoreFxNative")
3313 def fxBranch = getFxBranch(branch)
3314 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch)
3316 copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
3317 includePatterns('artifacts/bin/build.tar.gz')
3318 targetDirectory('bin/CoreFxNative')
3320 latestSuccessful(true)
3324 shell("mkdir ./bin/CoreFxBinDir")
3325 shell("tar -xf ./bin/CoreFxNative/artifacts/bin/build.tar.gz -C ./bin/CoreFxBinDir")
3328 if (isPmiAsmDiffsScenario) {
3329 shell("tar -xzf ./product.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0")
3330 shell("tar -xzf ./product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0")
3331 shell("tar -xzf ./coreroot.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0")
3332 shell("tar -xzf ./coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0")
3334 // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
3335 else if (doCoreFxTesting) {
3336 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
3337 shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
3340 def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration)
3341 shell("tar -xzf ./${testArtifactsTgzFileName} || exit 0") // extracts to ./bin/tests/${osGroup}.${architecture}.${configuration}
3344 // Execute the tests
3345 def runDocker = isNeedDocker(architecture, os, false)
3346 def dockerPrefix = ""
3349 def dockerImage = getDockerImageName(architecture, os, false)
3350 dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
3351 dockerCmd = dockerPrefix + "${dockerImage} "
3354 // If we are running a stress mode, we'll set those variables first.
3355 // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers.
3356 if (!doCoreFxTesting && isJitStressScenario(scenario)) {
3357 def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh"
3358 def envScriptCmds = envScriptCreate(os, scriptFileName)
3359 envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
3360 envScriptCmds += envScriptFinalize(os, scriptFileName)
3361 shell("${envScriptCmds}")
3362 testOpts += " --test-env=${scriptFileName}"
3365 // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet"
3366 // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The
3367 // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries.
3368 // On arm/arm64, it is not required as determining instruction boundaries is trivial.
3369 if (isGCStressRelatedTesting(scenario)) {
3370 if (architecture == 'x64') {
3371 shell('./init-tools.sh')
3375 if (isPmiAsmDiffsScenario) {
3377 python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} --skip_baseline_build""")
3379 shell("tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm")
3381 else if (doCoreFxTesting) {
3382 def exclusionRspPath = "\${WORKSPACE}/tests/scripts/run-corefx-tests-exclusions.txt"
3384 \${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release --exclusion-rsp-file ${exclusionRspPath}""")
3387 def runScript = "${dockerCmd}./tests/runtest.sh"
3391 ${lowerConfiguration} \\
3392 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
3393 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
3394 --limitedDumpGeneration ${testOpts}""")
3397 if (isGcReliabilityFramework(scenario)) {
3398 // runtest.sh doesn't actually execute the reliability framework - do it here.
3401 dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
3404 shell("export COMPlus_gcServer=1")
3408 shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
3413 // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
3414 if (os in ['Ubuntu']) {
3415 SummaryBuilder summaries = new SummaryBuilder()
3416 summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
3417 summaries.emit(newJob)
3420 if (isPmiAsmDiffsScenario) {
3422 Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.tgz")
3424 else if (doCoreFxTesting) {
3425 Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
3426 if ((os == "Ubuntu") && (architecture == 'arm')) {
3427 // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error:
3429 // [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed
3431 // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario.
3432 // This is tracked by: https://github.com/dotnet/coreclr/issues/19447.
3435 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml")
3439 Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
3445 def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3447 assert isCrossGenComparisonScenario(scenario)
3449 def osGroup = getOSGroup(os)
3450 def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
3452 def workspaceRelativeResultsDir = "_"
3453 def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}"
3455 def crossArch = "x64"
3456 def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}"
3458 def jobFolder = getJobFolder(scenario)
3459 def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
3461 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
3464 def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
3467 copyArtifacts(inputCoreCLRBuildName) {
3468 includePatterns("${workspaceRelativeArtifactsArchive}")
3470 buildNumber('${CORECLR_BUILD}')
3474 shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
3476 def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
3477 def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
3478 def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
3479 def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
3481 def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
3482 def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
3484 shell("mkdir -p ${workspaceRelativeNativeArchResultDir}")
3485 shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3486 shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}")
3488 shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}")
3492 Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
3493 Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
3498 // Create a test job that will be used by a flow job.
3499 // Returns the newly created job.
3500 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
3501 // and we want the views to be the minimal set of "top-level" jobs that represent all work.
3502 def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
3504 def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3507 if (windowsArmJob) {
3508 newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3510 else if (isCrossGenComparisonScenario(scenario)) {
3511 newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3514 newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3517 setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false
3519 if (scenario == 'jitdiff') {
3520 def osGroup = getOSGroup(os)
3521 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
3524 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
3525 setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
3530 // Create a flow job to tie together a build job with the given test job.
3531 // Returns the new flow job.
3532 def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName)
3534 // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
3535 // Linux CoreCLR test
3536 def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
3537 def jobFolder = getJobFolder(scenario)
3539 def newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) {
3541 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
3543 // And then build the test build
3544 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
3547 JobReport.Report.addReference(inputCoreCLRBuildName)
3548 JobReport.Report.addReference(fullTestJobName)
3550 addToViews(newFlowJob, true, isPR, architecture, os, configuration, scenario) // isFlowJob = true
3552 setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true
3554 Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
3555 addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
3560 // Determine if we should generate a flow job for the given parameters.
3561 // Returns true if the job should be generated.
3562 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
3564 // The various "innerloop" jobs are only available as PR triggered.
3567 if (isInnerloopTestScenario(scenario)) {
3571 if (scenario == 'corefx_innerloop') {
3576 // Disable flow jobs for innerloop pr.
3578 // The only exception is windows arm(64)
3579 if (isInnerloopTestScenario(scenario) && isPR && os != 'Windows_NT') {
3580 assert scenario != 'corefx_innerloop'
3585 // Filter based on OS and architecture.
3587 switch (architecture) {
3589 if (os != "Ubuntu" && os != "Windows_NT") {
3594 if (os != "Ubuntu16.04" && os != "Windows_NT") {
3599 if (os != "Ubuntu") {
3604 if (!(os in Constants.crossList)) {
3607 if (os == "Windows_NT") {
3615 println("Unknown architecture: ${architecture}")
3620 def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal')
3622 // Filter based on scenario in OS.
3624 if (os == 'Windows_NT') {
3625 assert architecture == 'arm' || architecture == 'arm64'
3626 if (!isArmWindowsScenario(scenario)) {
3629 if (isNormalOrInnerloop && (configuration == 'Debug')) {
3630 // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
3631 // To do that, it doesn't require a flow job.
3637 if (architecture == 'arm') {
3638 if (!(scenario in Constants.validLinuxArmScenarios)) {
3642 else if (architecture == 'arm64') {
3643 if (!(scenario in Constants.validLinuxArm64Scenarios)) {
3647 else if (architecture == 'x86') {
3648 // Linux/x86 only want innerloop and default test
3649 if (!isNormalOrInnerloop) {
3653 else if (architecture == 'x64') {
3654 // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which
3655 // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test
3656 // build from a Windows machine.
3657 if (isCoreFxScenario(scenario)) {
3663 // For CentOS, we only want Checked/Release builds.
3664 if (os == 'CentOS7.1') {
3665 if (configuration != 'Checked' && configuration != 'Release') {
3668 if (!isNormalOrInnerloop && !isR2RScenario(scenario)) {
3673 // For RedHat and Debian, we only do Release builds.
3674 else if (os == 'RHEL7.2' || os == 'Debian8.4') {
3675 if (configuration != 'Release') {
3678 if (!isNormalOrInnerloop) {
3683 // Next, filter based on scenario.
3685 if (isJitStressScenario(scenario)) {
3686 if (configuration != 'Checked') {
3690 else if (isR2RBaselineScenario(scenario)) {
3691 if (configuration != 'Checked' && configuration != 'Release') {
3695 else if (isR2RStressScenario(scenario)) {
3696 if (configuration != 'Checked') {
3700 else if (isCrossGenComparisonScenario(scenario)) {
3701 return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
3709 // Long GC tests take a long time on non-Release builds
3710 // ilrt is also Release only
3711 if (configuration != 'Release') {
3717 if (configuration != 'Checked') {
3722 case 'gc_reliability_framework':
3723 case 'standalone_gc':
3724 if (configuration != 'Release' && configuration != 'Checked') {
3733 if (os != 'Windows_NT' && os != 'Ubuntu') {
3743 if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) {
3748 case 'pmi_asm_diffs':
3749 if (configuration != 'Checked') {
3752 // No need for flow job except for Linux arm/arm64
3753 if ((os != 'Windows_NT') && (architecture != 'arm') && (architecture != 'arm64')) {
3758 case 'corefx_innerloop':
3759 // No flow job needed
3763 println("Unknown scenario: ${scenario}")
3769 // The job was not filtered out, so we should generate it!
3773 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows.
3774 Constants.allScenarios.each { scenario ->
3775 [true, false].each { isPR ->
3776 Constants.architectureList.each { architecture ->
3777 Constants.configurationList.each { configuration ->
3778 Constants.osList.each { os ->
3780 if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) {
3784 def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
3785 def doCoreFxTesting = isCoreFxScenario(scenario)
3786 def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
3787 def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs')
3789 // Figure out the job name of the CoreCLR build the test will depend on.
3791 def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
3792 def inputCoreCLRBuildIsBuildOnly = false
3793 if (doCoreFxTesting || isPmiAsmDiffsScenario) {
3794 // Every CoreFx test depends on its own unique build.
3795 inputCoreCLRBuildScenario = scenario
3796 if (windowsArmJob) {
3797 // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs.
3798 inputCoreCLRBuildIsBuildOnly = true
3801 else if (doCrossGenComparison) {
3802 inputCoreCLRBuildScenario = scenario
3805 def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
3806 def inputCoreCLRBuildName = projectFolder + '/' +
3807 Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
3809 // =============================================================================================
3810 // Create the test job
3811 // =============================================================================================
3813 def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
3815 // =============================================================================================
3816 // Create a build flow to join together the build and tests required to run this test.
3817 // =============================================================================================
3819 if (os == 'RHEL7.2' || os == 'Debian8.4') {
3820 // Do not create the flow job for RHEL jobs.
3824 def fullTestJobName = projectFolder + '/' + testJob.name
3825 def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName)
3833 JobReport.Report.generateJobReport(out)
3835 // Make the call to generate the help job
3836 Utilities.createHelperJob(this, project, branch,
3837 "Welcome to the ${project} Repository", // This is prepended to the help message
3838 "Have a nice day!") // This is appended to the help message. You might put known issues here.
3840 Utilities.addCROSSCheck(this, project, branch)