Add new robustness extension tests
[platform/upstream/VK-GL-CTS.git] / scripts / build_android_mustpass.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 from build.common import DEQP_DIR
24 from build.config import ANY_GENERATOR
25 from build_caselists import Module, getModuleByName, getBuildConfig, DEFAULT_BUILD_DIR, DEFAULT_TARGET
26 from mustpass import Project, Package, Mustpass, Configuration, include, exclude, genMustpassLists
27
28 import os
29
30 COPYRIGHT_DECLARATION = """
31      Copyright (C) 2016 The Android Open Source Project
32
33      Licensed under the Apache License, Version 2.0 (the "License");
34      you may not use this file except in compliance with the License.
35      You may obtain a copy of the License at
36
37           http://www.apache.org/licenses/LICENSE-2.0
38
39      Unless required by applicable law or agreed to in writing, software
40      distributed under the License is distributed on an "AS IS" BASIS,
41      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
42      See the License for the specific language governing permissions and
43      limitations under the License.
44      """
45
46 CTS_DATA_DIR                                    = os.path.join(DEQP_DIR, "android", "cts")
47
48 CTS_PROJECT                                             = Project(path = CTS_DATA_DIR, copyright = COPYRIGHT_DECLARATION)
49
50 EGL_MODULE                                              = getModuleByName("dEQP-EGL")
51 GLES2_MODULE                                    = getModuleByName("dEQP-GLES2")
52 GLES3_MODULE                                    = getModuleByName("dEQP-GLES3")
53 GLES31_MODULE                                   = getModuleByName("dEQP-GLES31")
54 VULKAN_MODULE                                   = getModuleByName("dEQP-VK")
55
56 # Lollipop
57
58 LMP_GLES3_PKG                                   = Package(module = GLES3_MODULE, configurations = [
59                 Configuration(name                      = "master",
60                                           glconfig              = "rgba8888d24s8ms0",
61                                           rotation              = "unspecified",
62                                           surfacetype   = "window",
63                                           filters               = [include("es30-lmp.txt")]),
64         ])
65 LMP_GLES31_PKG                                  = Package(module = GLES31_MODULE, configurations = [
66                 Configuration(name                      = "master",
67                                           glconfig              = "rgba8888d24s8ms0",
68                                           rotation              = "unspecified",
69                                           surfacetype   = "window",
70                                           filters               = [include("es31-lmp.txt")]),
71         ])
72
73 # Lollipop MR1
74
75 LMP_MR1_GLES3_PKG                               = Package(module = GLES3_MODULE, configurations = [
76                 Configuration(name                      = "master",
77                                           glconfig              = "rgba8888d24s8ms0",
78                                           rotation              = "unspecified",
79                                           surfacetype   = "window",
80                                           filters               = [include("es30-lmp-mr1.txt")]),
81         ])
82 LMP_MR1_GLES31_PKG                              = Package(module = GLES31_MODULE, configurations = [
83                 Configuration(name                      = "master",
84                                           glconfig              = "rgba8888d24s8ms0",
85                                           rotation              = "unspecified",
86                                           surfacetype   = "window",
87                                           filters               = [include("es31-lmp-mr1.txt")]),
88         ])
89
90 # Marshmallow
91
92 MNC_EGL_PKG                                             = Package(module = EGL_MODULE, configurations = [
93                 # Master
94                 Configuration(name                      = "master",
95                                           glconfig              = "rgba8888d24s8ms0",
96                                           rotation              = "unspecified",
97                                           surfacetype   = "window",
98                                           filters               = [include("egl-master.txt")]),
99         ])
100 MNC_GLES2_PKG                                   = Package(module = GLES2_MODULE, configurations = [
101                 # Master
102                 Configuration(name                      = "master",
103                                           glconfig              = "rgba8888d24s8ms0",
104                                           rotation              = "unspecified",
105                                           surfacetype   = "window",
106                                           filters               = [include("gles2-master.txt")]),
107         ])
108 MNC_GLES3_PKG                                   = Package(module = GLES3_MODULE, configurations = [
109                 # Master
110                 Configuration(name                      = "master",
111                                           glconfig              = "rgba8888d24s8ms0",
112                                           rotation              = "unspecified",
113                                           surfacetype   = "window",
114                                           filters               = [include("gles3-master.txt")]),
115                 # Rotations
116                 Configuration(name                      = "rotate-portrait",
117                                           glconfig              = "rgba8888d24s8ms0",
118                                           rotation              = "0",
119                                           surfacetype   = "window",
120                                           filters               = [include("gles3-master.txt"), include("gles3-rotation.txt")]),
121                 Configuration(name                      = "rotate-landscape",
122                                           glconfig              = "rgba8888d24s8ms0",
123                                           rotation              = "90",
124                                           surfacetype   = "window",
125                                           filters               = [include("gles3-master.txt"), include("gles3-rotation.txt")]),
126                 Configuration(name                      = "rotate-reverse-portrait",
127                                           glconfig              = "rgba8888d24s8ms0",
128                                           rotation              = "180",
129                                           surfacetype   = "window",
130                                           filters               = [include("gles3-master.txt"), include("gles3-rotation.txt")]),
131                 Configuration(name                      = "rotate-reverse-landscape",
132                                           glconfig              = "rgba8888d24s8ms0",
133                                           rotation              = "270",
134                                           surfacetype   = "window",
135                                           filters               = [include("gles3-master.txt"), include("gles3-rotation.txt")]),
136
137                 # MSAA
138                 Configuration(name                      = "multisample",
139                                           glconfig              = "rgba8888d24s8ms4",
140                                           rotation              = "unspecified",
141                                           surfacetype   = "window",
142                                           filters               = [include("gles3-master.txt"),
143                                                                            include("gles3-multisample.txt"),
144                                                                            exclude("gles3-multisample-issues.txt")]),
145
146                 # Pixel format
147                 Configuration(name                      = "565-no-depth-no-stencil",
148                                           glconfig              = "rgb565d0s0ms0",
149                                           rotation              = "unspecified",
150                                           surfacetype   = "window",
151                                           filters               = [include("gles3-master.txt"),
152                                                                            include("gles3-pixelformat.txt"),
153                                                                            exclude("gles3-pixelformat-issues.txt")]),
154         ])
155 MNC_GLES31_PKG                                  = Package(module = GLES31_MODULE, configurations = [
156                 # Master
157                 Configuration(name                      = "master",
158                                           glconfig              = "rgba8888d24s8ms0",
159                                           rotation              = "unspecified",
160                                           surfacetype   = "window",
161                                           filters               = [include("gles31-master.txt")]),
162
163                 # Rotations
164                 Configuration(name                      = "rotate-portrait",
165                                           glconfig              = "rgba8888d24s8ms0",
166                                           rotation              = "0",
167                                           surfacetype   = "window",
168                                           filters               = [include("gles31-master.txt"), include("gles31-rotation.txt")]),
169                 Configuration(name                      = "rotate-landscape",
170                                           glconfig              = "rgba8888d24s8ms0",
171                                           rotation              = "90",
172                                           surfacetype   = "window",
173                                           filters               = [include("gles31-master.txt"), include("gles31-rotation.txt")]),
174                 Configuration(name                      = "rotate-reverse-portrait",
175                                           glconfig              = "rgba8888d24s8ms0",
176                                           rotation              = "180",
177                                           surfacetype   = "window",
178                                           filters               = [include("gles31-master.txt"), include("gles31-rotation.txt")]),
179                 Configuration(name                      = "rotate-reverse-landscape",
180                                           glconfig              = "rgba8888d24s8ms0",
181                                           rotation              = "270",
182                                           surfacetype   = "window",
183                                           filters               = [include("gles31-master.txt"), include("gles31-rotation.txt")]),
184
185                 # MSAA
186                 Configuration(name                      = "multisample",
187                                           glconfig              = "rgba8888d24s8ms4",
188                                           rotation              = "unspecified",
189                                           surfacetype   = "window",
190                                           filters               = [include("gles31-master.txt"), include("gles31-multisample.txt")]),
191
192                 # Pixel format
193                 Configuration(name                      = "565-no-depth-no-stencil",
194                                           glconfig              = "rgb565d0s0ms0",
195                                           rotation              = "unspecified",
196                                           surfacetype   = "window",
197                                           filters               = [include("gles31-master.txt"), include("gles31-pixelformat.txt")]),
198         ])
199
200 # NYC
201
202 NYC_EGL_COMMON_FILTERS                  = [include("egl-master.txt")]
203 NYC_EGL_PKG                                             = Package(module = EGL_MODULE, configurations = [
204                 # Master
205                 Configuration(name                      = "master",
206                                           glconfig              = "rgba8888d24s8ms0",
207                                           rotation              = "unspecified",
208                                           surfacetype   = "window",
209                                           filters               = NYC_EGL_COMMON_FILTERS,
210                                       runtime           = "24m"),
211         ])
212
213 NYC_GLES2_COMMON_FILTERS                        = [
214                 include("gles2-master.txt")
215         ]
216 NYC_GLES2_PKG                                   = Package(module = GLES2_MODULE, configurations = [
217                 # Master
218                 Configuration(name                      = "master",
219                                           glconfig              = "rgba8888d24s8ms0",
220                                           rotation              = "unspecified",
221                                           surfacetype   = "window",
222                                           filters               = NYC_GLES2_COMMON_FILTERS,
223                                           runtime               = "40m"),
224         ])
225
226 NYC_GLES3_COMMON_FILTERS                = [
227                 include("gles3-master.txt")
228         ]
229 NYC_GLES3_PKG                                   = Package(module = GLES3_MODULE, configurations = [
230                 # Master
231                 Configuration(name                      = "master",
232                                           glconfig              = "rgba8888d24s8ms0",
233                                           rotation              = "unspecified",
234                                           surfacetype   = "window",
235                                           filters               = NYC_GLES3_COMMON_FILTERS,
236                                           runtime               = "1h15m"),
237                 # Rotations
238                 Configuration(name                      = "rotate-portrait",
239                                           glconfig              = "rgba8888d24s8ms0",
240                                           rotation              = "0",
241                                           surfacetype   = "window",
242                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
243                                           runtime               = "7m"),
244                 Configuration(name                      = "rotate-landscape",
245                                           glconfig              = "rgba8888d24s8ms0",
246                                           rotation              = "90",
247                                           surfacetype   = "window",
248                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
249                                           runtime               = "7m"),
250                 Configuration(name                      = "rotate-reverse-portrait",
251                                           glconfig              = "rgba8888d24s8ms0",
252                                           rotation              = "180",
253                                           surfacetype   = "window",
254                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
255                                           runtime               = "7m"),
256                 Configuration(name                      = "rotate-reverse-landscape",
257                                           glconfig              = "rgba8888d24s8ms0",
258                                           rotation              = "270",
259                                           surfacetype   = "window",
260                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
261                                           runtime               = "7m"),
262
263                 # MSAA
264                 Configuration(name                      = "multisample",
265                                           glconfig              = "rgba8888d24s8ms4",
266                                           rotation              = "unspecified",
267                                           surfacetype   = "window",
268                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt")],
269                                           runtime               = "10m"),
270
271                 # Pixel format
272                 Configuration(name                      = "565-no-depth-no-stencil",
273                                           glconfig              = "rgb565d0s0ms0",
274                                           rotation              = "unspecified",
275                                           surfacetype   = "window",
276                                           filters               = NYC_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt")],
277                                           runtime               = "10m"),
278         ])
279
280 NYC_GLES31_COMMON_FILTERS               = [
281                 include("gles31-master.txt")
282         ]
283 NYC_GLES31_PKG                                  = Package(module = GLES31_MODULE, configurations = [
284                 # Master
285                 Configuration(name                      = "master",
286                                           glconfig              = "rgba8888d24s8ms0",
287                                           rotation              = "unspecified",
288                                           surfacetype   = "window",
289                                           filters               = NYC_GLES31_COMMON_FILTERS,
290                                           runtime               = "7h30m"),
291
292                 # Rotations
293                 Configuration(name                      = "rotate-portrait",
294                                           glconfig              = "rgba8888d24s8ms0",
295                                           rotation              = "0",
296                                           surfacetype   = "window",
297                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
298                                           runtime               = "1m30s"),
299                 Configuration(name                      = "rotate-landscape",
300                                           glconfig              = "rgba8888d24s8ms0",
301                                           rotation              = "90",
302                                           surfacetype   = "window",
303                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
304                                           runtime               = "1m30s"),
305                 Configuration(name                      = "rotate-reverse-portrait",
306                                           glconfig              = "rgba8888d24s8ms0",
307                                           rotation              = "180",
308                                           surfacetype   = "window",
309                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
310                                           runtime               = "1m30s"),
311                 Configuration(name                      = "rotate-reverse-landscape",
312                                           glconfig              = "rgba8888d24s8ms0",
313                                           rotation              = "270",
314                                           surfacetype   = "window",
315                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
316                                           runtime               = "1m30s"),
317
318                 # MSAA
319                 Configuration(name                      = "multisample",
320                                           glconfig              = "rgba8888d24s8ms4",
321                                           rotation              = "unspecified",
322                                           surfacetype   = "window",
323                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")],
324                                           runtime               = "2m"),
325
326                 # Pixel format
327                 Configuration(name                      = "565-no-depth-no-stencil",
328                                           glconfig              = "rgb565d0s0ms0",
329                                           rotation              = "unspecified",
330                                           surfacetype   = "window",
331                                           filters               = NYC_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")],
332                                           runtime               = "1m"),
333         ])
334
335 NYC_VULKAN_FILTERS                              = [
336                 include("vk-master.txt")
337         ]
338 NYC_VULKAN_PKG                                  = Package(module = VULKAN_MODULE, configurations = [
339                 Configuration(name                      = "master",
340                                           filters               = NYC_VULKAN_FILTERS,
341                                           runtime               = "3h45m"),
342         ])
343
344 # Master
345
346 MASTER_EGL_COMMON_FILTERS               = [include("egl-master.txt"),
347                                                                    exclude("egl-test-issues.txt"),
348                                                                    exclude("egl-internal-api-tests.txt"),
349                                                                    exclude("egl-manual-robustness.txt")]
350 MASTER_EGL_PKG                                  = Package(module = EGL_MODULE, configurations = [
351                 # Master
352                 Configuration(name                      = "master",
353                                           glconfig              = "rgba8888d24s8ms0",
354                                           rotation              = "unspecified",
355                                           surfacetype   = "window",
356                                           required              = True,
357                                           filters               = MASTER_EGL_COMMON_FILTERS,
358                                       runtime           = "24m"),
359         ])
360
361 MASTER_GLES2_COMMON_FILTERS             = [
362                 include("gles2-master.txt"),
363                 exclude("gles2-test-issues.txt"),
364                 exclude("gles2-failures.txt")
365         ]
366 MASTER_GLES2_PKG                                = Package(module = GLES2_MODULE, configurations = [
367                 # Master
368                 Configuration(name                      = "master",
369                                           glconfig              = "rgba8888d24s8ms0",
370                                           rotation              = "unspecified",
371                                           surfacetype   = "window",
372                                           required              = True,
373                                           filters               = MASTER_GLES2_COMMON_FILTERS,
374                                           runtime               = "40m"),
375         ])
376
377 MASTER_GLES3_COMMON_FILTERS             = [
378                 include("gles3-master.txt"),
379                 exclude("gles3-hw-issues.txt"),
380                 exclude("gles3-driver-issues.txt"),
381                 exclude("gles3-test-issues.txt"),
382                 exclude("gles3-spec-issues.txt")
383         ]
384 MASTER_GLES3_PKG                                = Package(module = GLES3_MODULE, configurations = [
385                 # Master
386                 Configuration(name                      = "master",
387                                           glconfig              = "rgba8888d24s8ms0",
388                                           rotation              = "unspecified",
389                                           surfacetype   = "window",
390                                           required              = True,
391                                           filters               = MASTER_GLES3_COMMON_FILTERS,
392                                           runtime               = "1h15m"),
393                 # Rotations
394                 Configuration(name                      = "rotate-portrait",
395                                           glconfig              = "rgba8888d24s8ms0",
396                                           rotation              = "0",
397                                           surfacetype   = "window",
398                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
399                                           runtime               = "7m"),
400                 Configuration(name                      = "rotate-landscape",
401                                           glconfig              = "rgba8888d24s8ms0",
402                                           rotation              = "90",
403                                           surfacetype   = "window",
404                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
405                                           runtime               = "7m"),
406                 Configuration(name                      = "rotate-reverse-portrait",
407                                           glconfig              = "rgba8888d24s8ms0",
408                                           rotation              = "180",
409                                           surfacetype   = "window",
410                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
411                                           runtime               = "7m"),
412                 Configuration(name                      = "rotate-reverse-landscape",
413                                           glconfig              = "rgba8888d24s8ms0",
414                                           rotation              = "270",
415                                           surfacetype   = "window",
416                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")],
417                                           runtime               = "7m"),
418
419                 # MSAA
420                 Configuration(name                      = "multisample",
421                                           glconfig              = "rgba8888d24s8ms4",
422                                           rotation              = "unspecified",
423                                           surfacetype   = "window",
424                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt"),
425                                                                                                                                          exclude("gles3-multisample-issues.txt")],
426                                           runtime               = "10m"),
427
428                 # Pixel format
429                 Configuration(name                      = "565-no-depth-no-stencil",
430                                           glconfig              = "rgb565d0s0ms0",
431                                           rotation              = "unspecified",
432                                           surfacetype   = "window",
433                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt"),
434                                                                                                                                          exclude("gles3-pixelformat-issues.txt")],
435                                           runtime               = "10m"),
436         ])
437
438 MASTER_GLES31_COMMON_FILTERS    = [
439                 include("gles31-master.txt"),
440                 exclude("gles31-hw-issues.txt"),
441                 exclude("gles31-driver-issues.txt"),
442                 exclude("gles31-test-issues.txt"),
443                 exclude("gles31-spec-issues.txt"),
444         ]
445 MASTER_GLES31_PKG                               = Package(module = GLES31_MODULE, configurations = [
446                 # Master
447                 Configuration(name                      = "master",
448                                           glconfig              = "rgba8888d24s8ms0",
449                                           rotation              = "unspecified",
450                                           surfacetype   = "window",
451                                           required              = True,
452                                           filters               = MASTER_GLES31_COMMON_FILTERS,
453                                           runtime               = "7h30m"),
454
455                 # Rotations
456                 Configuration(name                      = "rotate-portrait",
457                                           glconfig              = "rgba8888d24s8ms0",
458                                           rotation              = "0",
459                                           surfacetype   = "window",
460                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
461                                           runtime               = "1m30s"),
462                 Configuration(name                      = "rotate-landscape",
463                                           glconfig              = "rgba8888d24s8ms0",
464                                           rotation              = "90",
465                                           surfacetype   = "window",
466                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
467                                           runtime               = "1m30s"),
468                 Configuration(name                      = "rotate-reverse-portrait",
469                                           glconfig              = "rgba8888d24s8ms0",
470                                           rotation              = "180",
471                                           surfacetype   = "window",
472                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
473                                           runtime               = "1m30s"),
474                 Configuration(name                      = "rotate-reverse-landscape",
475                                           glconfig              = "rgba8888d24s8ms0",
476                                           rotation              = "270",
477                                           surfacetype   = "window",
478                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")],
479                                           runtime               = "1m30s"),
480
481                 # MSAA
482                 Configuration(name                      = "multisample",
483                                           glconfig              = "rgba8888d24s8ms4",
484                                           rotation              = "unspecified",
485                                           surfacetype   = "window",
486                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")],
487                                           runtime               = "2m"),
488
489                 # Pixel format
490                 Configuration(name                      = "565-no-depth-no-stencil",
491                                           glconfig              = "rgb565d0s0ms0",
492                                           rotation              = "unspecified",
493                                           surfacetype   = "window",
494                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")],
495                                           runtime               = "1m"),
496         ])
497
498 MASTER_VULKAN_FILTERS                   = [
499                 include("vk-master.txt"),
500                 exclude("vk-not-applicable.txt"),
501                 exclude("vk-excluded-tests.txt"),
502                 exclude("vk-test-issues.txt"),
503         ]
504 MASTER_VULKAN_PKG                               = Package(module = VULKAN_MODULE, configurations = [
505                 Configuration(name                      = "master",
506                                           filters               = MASTER_VULKAN_FILTERS,
507                                           runtime               = "3h45m"),
508         ])
509
510 MUSTPASS_LISTS                          = [
511                 Mustpass(project = CTS_PROJECT, version = "lmp",                packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]),
512                 Mustpass(project = CTS_PROJECT, version = "lmp-mr1",    packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]),
513                 Mustpass(project = CTS_PROJECT, version = "mnc",                packages = [MNC_EGL_PKG, MNC_GLES2_PKG, MNC_GLES3_PKG, MNC_GLES31_PKG]),
514                 Mustpass(project = CTS_PROJECT, version = "nyc",                packages = [NYC_EGL_PKG, NYC_GLES2_PKG, NYC_GLES3_PKG, NYC_GLES31_PKG, NYC_VULKAN_PKG]),
515                 Mustpass(project = CTS_PROJECT, version = "master",             packages = [MASTER_EGL_PKG, MASTER_GLES2_PKG, MASTER_GLES3_PKG, MASTER_GLES31_PKG, MASTER_VULKAN_PKG])
516         ]
517
518 BUILD_CONFIG                            = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug")
519
520 if __name__ == "__main__":
521         genMustpassLists(MUSTPASS_LISTS, ANY_GENERATOR, BUILD_CONFIG)