2 * Copyright © 2009-2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 * Tvrtko Ursulin <tvrtko.ursulin@intel.com>
30 /** @file gem_userptr_blits.c
32 * This is a test of doing many blits using a mixture of normal system pages
33 * and uncached linear buffers, with a working set larger than the
36 * The goal is to simply ensure the basics work.
55 #include "intel_bufmgr.h"
56 #include "intel_batchbuffer.h"
57 #include "intel_chipset.h"
58 #include "ioctl_wrappers.h"
60 #include "eviction_common.c"
63 #define PAGE_SIZE 4096
66 #define LOCAL_I915_GEM_USERPTR 0x33
67 #define LOCAL_IOCTL_I915_GEM_USERPTR DRM_IOWR (DRM_COMMAND_BASE + LOCAL_I915_GEM_USERPTR, struct local_i915_gem_userptr)
68 struct local_i915_gem_userptr {
72 #define LOCAL_I915_USERPTR_READ_ONLY (1<<0)
73 #define LOCAL_I915_USERPTR_UNSYNCHRONIZED (1<<31)
77 static uint32_t userptr_flags = LOCAL_I915_USERPTR_UNSYNCHRONIZED;
82 static uint32_t linear[WIDTH*HEIGHT];
84 static void gem_userptr_test_unsynchronized(void)
86 userptr_flags = LOCAL_I915_USERPTR_UNSYNCHRONIZED;
89 static void gem_userptr_test_synchronized(void)
94 static int gem_userptr(int fd, void *ptr, int size, int read_only, uint32_t *handle)
96 struct local_i915_gem_userptr userptr;
99 memset(&userptr, 0, sizeof(userptr));
100 userptr.user_ptr = (uintptr_t)ptr;
101 userptr.user_size = size;
102 userptr.flags = userptr_flags;
104 userptr.flags |= LOCAL_I915_USERPTR_READ_ONLY;
106 ret = drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &userptr);
109 igt_skip_on_f(ret == ENODEV &&
110 (userptr_flags & LOCAL_I915_USERPTR_UNSYNCHRONIZED) == 0 &&
112 "Skipping, synchronized mappings with no kernel CONFIG_MMU_NOTIFIER?");
114 *handle = userptr.handle;
120 static void gem_userptr_sync(int fd, uint32_t handle)
122 gem_set_domain(fd, handle, I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
126 copy(int fd, uint32_t dst, uint32_t src, unsigned int error)
129 struct drm_i915_gem_relocation_entry reloc[2];
130 struct drm_i915_gem_exec_object2 obj[3];
131 struct drm_i915_gem_execbuffer2 exec;
135 batch[i++] = XY_SRC_COPY_BLT_CMD |
136 XY_SRC_COPY_BLT_WRITE_ALPHA |
137 XY_SRC_COPY_BLT_WRITE_RGB;
138 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
143 batch[i++] = (3 << 24) | /* 32 bits */
144 (0xcc << 16) | /* copy ROP */
146 batch[i++] = 0; /* dst x1,y1 */
147 batch[i++] = (HEIGHT << 16) | WIDTH; /* dst x2,y2 */
148 batch[i++] = 0; /* dst reloc */
149 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
151 batch[i++] = 0; /* src x1,y1 */
152 batch[i++] = WIDTH*4;
153 batch[i++] = 0; /* src reloc */
154 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
156 batch[i++] = MI_BATCH_BUFFER_END;
157 batch[i++] = MI_NOOP;
159 handle = gem_create(fd, 4096);
160 gem_write(fd, handle, 0, batch, sizeof(batch));
162 reloc[0].target_handle = dst;
164 reloc[0].offset = 4 * sizeof(batch[0]);
165 reloc[0].presumed_offset = 0;
166 reloc[0].read_domains = I915_GEM_DOMAIN_RENDER;
167 reloc[0].write_domain = I915_GEM_DOMAIN_RENDER;
169 reloc[1].target_handle = src;
171 reloc[1].offset = 7 * sizeof(batch[0]);
172 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
173 reloc[1].offset += sizeof(batch[0]);
174 reloc[1].presumed_offset = 0;
175 reloc[1].read_domains = I915_GEM_DOMAIN_RENDER;
176 reloc[1].write_domain = 0;
178 memset(obj, 0, sizeof(obj));
179 exec.buffer_count = 0;
180 obj[exec.buffer_count++].handle = dst;
182 obj[exec.buffer_count++].handle = src;
183 obj[exec.buffer_count].handle = handle;
184 obj[exec.buffer_count].relocation_count = 2;
185 obj[exec.buffer_count].relocs_ptr = (uintptr_t)reloc;
187 exec.buffers_ptr = (uintptr_t)obj;
189 exec.batch_start_offset = 0;
190 exec.batch_len = i * 4;
191 exec.DR1 = exec.DR4 = 0;
192 exec.num_cliprects = 0;
193 exec.cliprects_ptr = 0;
194 exec.flags = HAS_BLT_RING(intel_get_drm_devid(fd)) ? I915_EXEC_BLT : 0;
195 i915_execbuffer2_set_context_id(exec, 0);
198 ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, &exec);
203 igt_assert(ret != 0);
205 igt_assert(ret == error);
207 gem_close(fd, handle);
211 blit(int fd, uint32_t dst, uint32_t src, uint32_t *all_bo, int n_bo)
214 struct drm_i915_gem_relocation_entry reloc[2];
215 struct drm_i915_gem_exec_object2 *obj;
216 struct drm_i915_gem_execbuffer2 exec;
220 batch[i++] = XY_SRC_COPY_BLT_CMD |
221 XY_SRC_COPY_BLT_WRITE_ALPHA |
222 XY_SRC_COPY_BLT_WRITE_RGB;
223 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
227 batch[i++] = (3 << 24) | /* 32 bits */
228 (0xcc << 16) | /* copy ROP */
230 batch[i++] = 0; /* dst x1,y1 */
231 batch[i++] = (HEIGHT << 16) | WIDTH; /* dst x2,y2 */
232 batch[i++] = 0; /* dst reloc */
233 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
235 batch[i++] = 0; /* src x1,y1 */
236 batch[i++] = WIDTH*4;
237 batch[i++] = 0; /* src reloc */
238 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
240 batch[i++] = MI_BATCH_BUFFER_END;
241 batch[i++] = MI_NOOP;
243 handle = gem_create(fd, 4096);
244 gem_write(fd, handle, 0, batch, sizeof(batch));
246 reloc[0].target_handle = dst;
248 reloc[0].offset = 4 * sizeof(batch[0]);
249 reloc[0].presumed_offset = 0;
250 reloc[0].read_domains = I915_GEM_DOMAIN_RENDER;
251 reloc[0].write_domain = I915_GEM_DOMAIN_RENDER;
253 reloc[1].target_handle = src;
255 reloc[1].offset = 7 * sizeof(batch[0]);
256 if (intel_gen(intel_get_drm_devid(fd)) >= 8)
257 reloc[1].offset += sizeof(batch[0]);
258 reloc[1].presumed_offset = 0;
259 reloc[1].read_domains = I915_GEM_DOMAIN_RENDER;
260 reloc[1].write_domain = 0;
262 obj = calloc(n_bo + 1, sizeof(*obj));
263 for (n = 0; n < n_bo; n++)
264 obj[n].handle = all_bo[n];
265 obj[n].handle = handle;
266 obj[n].relocation_count = 2;
267 obj[n].relocs_ptr = (uintptr_t)reloc;
269 exec.buffers_ptr = (uintptr_t)obj;
270 exec.buffer_count = n_bo + 1;
271 exec.batch_start_offset = 0;
272 exec.batch_len = i * 4;
273 exec.DR1 = exec.DR4 = 0;
274 exec.num_cliprects = 0;
275 exec.cliprects_ptr = 0;
276 exec.flags = HAS_BLT_RING(intel_get_drm_devid(fd)) ? I915_EXEC_BLT : 0;
277 i915_execbuffer2_set_context_id(exec, 0);
280 ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, &exec);
284 gem_close(fd, handle);
291 create_userptr(int fd, uint32_t val, uint32_t *ptr)
296 ret = gem_userptr(fd, ptr, sizeof(linear), 0, &handle);
297 igt_assert(ret == 0);
298 igt_assert(handle != 0);
300 /* Fill the BO with dwords starting at val */
301 for (i = 0; i < WIDTH*HEIGHT; i++)
307 static void **handle_ptr_map;
308 static unsigned *handle_size_map;
309 static unsigned int num_handle_map;
311 static void reset_handle_ptr(void)
313 if (num_handle_map == 0)
316 free(handle_ptr_map);
317 handle_ptr_map = NULL;
319 free(handle_size_map);
320 handle_size_map = NULL;
325 static void add_handle_ptr(uint32_t handle, void *ptr, int size)
327 if (handle >= num_handle_map) {
328 int max = (4096 + handle) & -4096;
330 handle_ptr_map = realloc(handle_ptr_map,
331 max * sizeof(void*));
332 igt_assert(handle_ptr_map);
333 memset(handle_ptr_map + num_handle_map, 0,
334 (max - num_handle_map) * sizeof(void*));
336 handle_size_map = realloc(handle_size_map,
337 max * sizeof(unsigned));
338 igt_assert(handle_size_map);
339 memset(handle_ptr_map + num_handle_map, 0,
340 (max - num_handle_map) * sizeof(unsigned));
342 num_handle_map = max;
345 handle_ptr_map[handle] = ptr;
346 handle_size_map[handle] = size;
349 static void *get_handle_ptr(uint32_t handle)
351 igt_assert(handle < num_handle_map);
352 return handle_ptr_map[handle];
355 static void free_handle_ptr(uint32_t handle)
357 igt_assert(handle < num_handle_map);
358 igt_assert(handle_ptr_map[handle]);
360 munmap(handle_ptr_map[handle], handle_size_map[handle]);
361 handle_ptr_map[handle] = NULL;
364 static uint32_t create_userptr_bo(int fd, int size)
370 ptr = mmap(NULL, size,
371 PROT_READ | PROT_WRITE,
372 MAP_ANONYMOUS | MAP_SHARED,
374 igt_assert(ptr != MAP_FAILED);
376 ret = gem_userptr(fd, (uint32_t *)ptr, size, 0, &handle);
377 igt_assert(ret == 0);
378 add_handle_ptr(handle, ptr, size);
383 static void flink_userptr_bo(uint32_t old_handle, uint32_t new_handle)
385 igt_assert(old_handle < num_handle_map);
386 igt_assert(handle_ptr_map[old_handle]);
388 add_handle_ptr(new_handle,
389 handle_ptr_map[old_handle],
390 handle_size_map[old_handle]);
393 static void clear(int fd, uint32_t handle, int size)
395 void *ptr = get_handle_ptr(handle);
397 igt_assert(ptr != NULL);
399 memset(ptr, 0, size);
402 static void free_userptr_bo(int fd, uint32_t handle)
404 gem_close(fd, handle);
405 free_handle_ptr(handle);
409 create_bo(int fd, uint32_t val)
414 handle = gem_create(fd, sizeof(linear));
416 /* Fill the BO with dwords starting at val */
417 for (i = 0; i < WIDTH*HEIGHT; i++)
419 gem_write(fd, handle, 0, linear, sizeof(linear));
425 check_cpu(uint32_t *ptr, uint32_t val)
429 for (i = 0; i < WIDTH*HEIGHT; i++) {
430 igt_assert_f(ptr[i] == val,
431 "Expected 0x%08x, found 0x%08x "
432 "at offset 0x%08x\n",
439 check_gpu(int fd, uint32_t handle, uint32_t val)
441 gem_read(fd, handle, 0, linear, sizeof(linear));
442 check_cpu(linear, val);
445 static int has_userptr(int fd)
452 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE) == 0);
453 oldflags = userptr_flags;
454 gem_userptr_test_unsynchronized();
455 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle);
456 userptr_flags = oldflags;
462 gem_close(fd, handle);
468 static int test_input_checking(int fd)
470 struct local_i915_gem_userptr userptr;
474 memset(&userptr, 0, sizeof(userptr));
475 userptr.user_ptr = 0;
476 userptr.user_size = 0;
478 ret = drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &userptr);
479 igt_assert(ret != 0);
482 memset(&userptr, 0, sizeof(userptr));
483 userptr.user_ptr = 0;
484 userptr.user_size = ~0;
486 ret = drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &userptr);
487 igt_assert(ret != 0);
490 memset(&userptr, 0, sizeof(userptr));
491 userptr.user_ptr = 0;
492 userptr.user_size = ~0;
494 ret = drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &userptr);
495 igt_assert(ret != 0);
500 static int test_access_control(int fd)
509 /* CAP_SYS_ADMIN is needed for UNSYNCHRONIZED mappings. */
510 gem_userptr_test_unsynchronized();
512 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE) == 0);
514 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle);
516 gem_close(fd, handle);
518 igt_assert(ret == EPERM);
526 static int test_invalid_null_pointer(int fd)
532 ret = gem_userptr(fd, NULL, PAGE_SIZE, 0, &handle);
533 igt_assert(ret == 0);
535 copy(fd, handle, handle, ~0); /* QQQ Precise errno? */
536 gem_close(fd, handle);
541 static int test_invalid_gtt_mapping(int fd)
543 uint32_t handle, handle2;
548 handle = create_bo(fd, 0);
549 ptr = gem_mmap__gtt(fd, handle, sizeof(linear), PROT_READ | PROT_WRITE);
550 gem_close(fd, handle);
551 igt_assert(ptr != NULL);
552 igt_assert(((unsigned long)ptr & (PAGE_SIZE - 1)) == 0);
553 igt_assert((sizeof(linear) & (PAGE_SIZE - 1)) == 0);
555 ret = gem_userptr(fd, ptr, sizeof(linear), 0, &handle2);
556 igt_assert(ret == 0);
557 copy(fd, handle2, handle2, ~0); /* QQQ Precise errno? */
558 gem_close(fd, handle2);
560 munmap(ptr, sizeof(linear));
565 #define PE_GTT_MAP 0x1
567 static void test_process_exit(int fd, int flags)
572 handle = create_userptr_bo(fd, sizeof(linear));
574 if (flags & PE_GTT_MAP) {
575 uint32_t *ptr = gem_mmap__gtt(fd, handle, sizeof(linear), PROT_READ | PROT_WRITE);
581 copy(fd, handle, handle, 0);
586 static void test_forked_access(int fd)
588 uint32_t handle1 = 0, handle2 = 0;
589 void *ptr1 = NULL, *ptr2 = NULL;
592 ret = posix_memalign(&ptr1, PAGE_SIZE, sizeof(linear));
593 ret |= madvise(ptr1, sizeof(linear), MADV_DONTFORK);
594 ret |= gem_userptr(fd, ptr1, sizeof(linear), 0, &handle1);
595 igt_assert(ret == 0);
599 ret = posix_memalign(&ptr2, PAGE_SIZE, sizeof(linear));
600 ret |= madvise(ptr2, sizeof(linear), MADV_DONTFORK);
601 ret |= gem_userptr(fd, ptr2, sizeof(linear), 0, &handle2);
602 igt_assert(ret == 0);
606 memset(ptr1, 0x1, sizeof(linear));
607 memset(ptr2, 0x2, sizeof(linear));
610 copy(fd, handle1, handle2, 0);
614 gem_userptr_sync(fd, handle1);
615 gem_userptr_sync(fd, handle2);
617 gem_close(fd, handle1);
618 gem_close(fd, handle2);
620 igt_assert(memcmp(ptr1, ptr2, sizeof(linear)) == 0);
622 ret = madvise(ptr1, sizeof(linear), MADV_DOFORK);
623 igt_assert(ret == 0);
626 ret = madvise(ptr2, sizeof(linear), MADV_DOFORK);
627 igt_assert(ret == 0);
631 static int test_forbidden_ops(int fd)
633 struct drm_i915_gem_pread gem_pread;
634 struct drm_i915_gem_pwrite gem_pwrite;
640 memset(&gem_pread, 0, sizeof(gem_pread));
641 memset(&gem_pwrite, 0, sizeof(gem_pwrite));
643 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE) == 0);
645 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle);
646 igt_assert(ret == 0);
648 /* pread/pwrite are not always forbidden, but when they
649 * are they should fail with EINVAL.
652 gem_pread.handle = handle;
653 gem_pread.offset = 0;
654 gem_pread.size = PAGE_SIZE;
655 gem_pread.data_ptr = (uintptr_t)buf;
656 ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_PREAD, &gem_pread);
657 igt_assert(ret == 0 || errno == EINVAL);
659 gem_pwrite.handle = handle;
660 gem_pwrite.offset = 0;
661 gem_pwrite.size = PAGE_SIZE;
662 gem_pwrite.data_ptr = (uintptr_t)buf;
663 ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &gem_pwrite);
664 igt_assert(ret == 0 || errno == EINVAL);
665 gem_close(fd, handle);
671 static unsigned char counter;
673 static void (* volatile orig_sigbus)(int sig, siginfo_t *info, void *param);
674 static volatile unsigned long sigbus_start;
675 static volatile long sigbus_cnt = -1;
678 check_bo(int fd1, uint32_t handle1, int is_userptr, int fd2, uint32_t handle2)
680 unsigned char *ptr1, *ptr2;
681 unsigned long size = sizeof(linear);
683 ptr2 = gem_mmap(fd2, handle2, sizeof(linear), PROT_READ | PROT_WRITE);
686 ptr1 = is_userptr > 0 ? get_handle_ptr(handle1) : ptr2;
688 ptr1 = gem_mmap(fd1, handle1, sizeof(linear), PROT_READ | PROT_WRITE);
693 sigbus_start = (unsigned long)ptr2;
694 igt_assert(memcmp(ptr1, ptr2, sizeof(linear)) == 0);
698 memset(ptr1, counter, size);
699 memset(ptr2, counter, size);
702 munmap(ptr1, sizeof(linear));
703 munmap(ptr2, sizeof(linear));
706 static int export_handle(int fd, uint32_t handle, int *outfd)
708 struct drm_prime_handle args;
711 args.handle = handle;
712 args.flags = DRM_CLOEXEC;
715 ret = drmIoctl(fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
723 static void sigbus(int sig, siginfo_t *info, void *param)
725 unsigned long ptr = (unsigned long)info->si_addr;
728 if (ptr >= sigbus_start &&
729 ptr < sigbus_start + sizeof(linear)) {
730 /* replace mapping to allow progress */
731 munmap((void *)sigbus_start, sizeof(linear));
732 addr = mmap((void *)sigbus_start, sizeof(linear),
733 PROT_READ | PROT_WRITE,
734 MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
735 igt_assert((unsigned long)addr == sigbus_start);
736 memset(addr, counter, sizeof(linear));
743 orig_sigbus(sig, info, param);
747 static int test_dmabuf(void)
750 uint32_t handle, handle_import;
753 struct sigaction sigact, orig_sigact;
755 fd1 = drm_open_any();
757 handle = create_userptr_bo(fd1, sizeof(linear));
758 memset(get_handle_ptr(handle), counter, sizeof(linear));
760 ret = export_handle(fd1, handle, &dma_buf_fd);
761 if (userptr_flags & LOCAL_I915_USERPTR_UNSYNCHRONIZED && ret) {
762 igt_assert(ret == EINVAL || ret == ENODEV);
763 free_userptr_bo(fd1, handle);
767 igt_assert(ret == 0);
768 igt_assert(dma_buf_fd >= 0);
771 fd2 = drm_open_any();
772 handle_import = prime_fd_to_handle(fd2, dma_buf_fd);
773 check_bo(fd1, handle, 1, fd2, handle_import);
775 /* close dma_buf, check whether nothing disappears. */
777 check_bo(fd1, handle, 1, fd2, handle_import);
779 /* destroy userptr object and expect SIGBUS */
780 free_userptr_bo(fd1, handle);
783 memset(&sigact, 0, sizeof(sigact));
784 sigact.sa_sigaction = sigbus;
785 sigact.sa_flags = SA_SIGINFO;
786 ret = sigaction(SIGBUS, &sigact, &orig_sigact);
787 igt_assert(ret == 0);
789 orig_sigbus = orig_sigact.sa_sigaction;
792 check_bo(fd2, handle_import, -1, fd2, handle_import);
793 igt_assert(sigbus_cnt > 0);
795 ret = sigaction(SIGBUS, &orig_sigact, NULL);
796 igt_assert(ret == 0);
804 static int test_usage_restrictions(int fd)
810 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE * 2) == 0);
812 /* Address not aligned. */
813 ret = gem_userptr(fd, (char *)ptr + 1, PAGE_SIZE, 0, &handle);
814 igt_assert(ret != 0);
816 /* Size not rounded to page size. */
817 ret = gem_userptr(fd, ptr, PAGE_SIZE - 1, 0, &handle);
818 igt_assert(ret != 0);
821 ret = gem_userptr(fd, (char *)ptr + 1, PAGE_SIZE - 1, 0, &handle);
822 igt_assert(ret != 0);
824 /* Read-only not supported. */
825 ret = gem_userptr(fd, (char *)ptr, PAGE_SIZE, 1, &handle);
826 igt_assert(ret != 0);
833 static int test_create_destroy(int fd)
839 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE) == 0);
841 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle);
842 igt_assert(ret == 0);
844 gem_close(fd, handle);
850 static int test_coherency(int fd, int count)
853 uint32_t *cpu, *cpu_val;
854 uint32_t *gpu, *gpu_val;
858 igt_require(intel_check_memory(count, sizeof(linear), CHECK_RAM));
859 igt_info("Using 2x%d 1MiB buffers\n", count);
861 ret = posix_memalign((void **)&memory, PAGE_SIZE, count*sizeof(linear));
862 igt_assert(ret == 0 && memory);
864 gpu = malloc(sizeof(uint32_t)*count*4);
865 gpu_val = gpu + count;
866 cpu = gpu_val + count;
867 cpu_val = cpu + count;
869 for (i = 0; i < count; i++) {
870 gpu[i] = create_bo(fd, start);
872 start += WIDTH*HEIGHT;
875 for (i = 0; i < count; i++) {
876 cpu[i] = create_userptr(fd, start, memory+i*WIDTH*HEIGHT);
878 start += WIDTH*HEIGHT;
881 igt_info("Verifying initialisation...\n");
882 for (i = 0; i < count; i++) {
883 check_gpu(fd, gpu[i], gpu_val[i]);
884 check_cpu(memory+i*WIDTH*HEIGHT, cpu_val[i]);
887 igt_info("Cyclic blits cpu->gpu, forward...\n");
888 for (i = 0; i < count * 4; i++) {
890 int dst = (i + 1) % count;
892 copy(fd, gpu[dst], cpu[src], 0);
893 gpu_val[dst] = cpu_val[src];
895 for (i = 0; i < count; i++)
896 check_gpu(fd, gpu[i], gpu_val[i]);
898 igt_info("Cyclic blits gpu->cpu, backward...\n");
899 for (i = 0; i < count * 4; i++) {
900 int src = (i + 1) % count;
903 copy(fd, cpu[dst], gpu[src], 0);
904 cpu_val[dst] = gpu_val[src];
906 for (i = 0; i < count; i++) {
907 gem_userptr_sync(fd, cpu[i]);
908 check_cpu(memory+i*WIDTH*HEIGHT, cpu_val[i]);
911 igt_info("Random blits...\n");
912 for (i = 0; i < count * 4; i++) {
913 int src = random() % count;
914 int dst = random() % count;
917 copy(fd, gpu[dst], cpu[src], 0);
918 gpu_val[dst] = cpu_val[src];
920 copy(fd, cpu[dst], gpu[src], 0);
921 cpu_val[dst] = gpu_val[src];
924 for (i = 0; i < count; i++) {
925 check_gpu(fd, gpu[i], gpu_val[i]);
926 gem_close(fd, gpu[i]);
928 gem_userptr_sync(fd, cpu[i]);
929 check_cpu(memory+i*WIDTH*HEIGHT, cpu_val[i]);
930 gem_close(fd, cpu[i]);
939 static struct igt_eviction_test_ops fault_ops = {
940 .create = create_userptr_bo,
941 .flink = flink_userptr_bo,
942 .close = free_userptr_bo,
947 static int can_swap(void)
949 unsigned long as, ram;
951 /* Cannot swap if not enough address space */
953 /* FIXME: Improve check criteria. */
954 if (sizeof(void*) < 8)
957 as = 256 * 1024; /* Just a big number */
959 ram = intel_get_total_ram_mb();
961 if ((as - 128) < (ram - 256))
967 #define min(a, b) ((a) < (b) ? (a) : (b))
969 static void test_forking_evictions(int fd, int size, int count,
975 trash_count = intel_get_total_ram_mb() * 11 / 10;
976 /* Use the fact test will spawn a number of child
977 * processes meaning swapping will be triggered system
978 * wide even if one process on it's own can't do it.
980 num_threads = min(sysconf(_SC_NPROCESSORS_ONLN) * 4, 12);
981 trash_count /= num_threads;
982 if (count > trash_count)
985 forking_evictions(fd, &fault_ops, size, count, trash_count, flags);
989 static void test_swapping_evictions(int fd, int size, int count)
993 igt_skip_on_f(!can_swap(),
994 "Not enough process address space for swapping tests.\n");
996 trash_count = intel_get_total_ram_mb() * 11 / 10;
998 swapping_evictions(fd, &fault_ops, size, count, trash_count);
1002 static void test_minor_evictions(int fd, int size, int count)
1004 minor_evictions(fd, &fault_ops, size, count);
1008 static void test_major_evictions(int fd, int size, int count)
1010 major_evictions(fd, &fault_ops, size, count);
1014 static void test_overlap(int fd, int expected)
1018 uint32_t handle, handle2;
1020 igt_assert(posix_memalign((void *)&ptr, PAGE_SIZE, PAGE_SIZE * 3) == 0);
1022 ret = gem_userptr(fd, ptr + PAGE_SIZE, PAGE_SIZE, 0, &handle);
1023 igt_assert(ret == 0);
1025 /* before, no overlap */
1026 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle2);
1028 gem_close(fd, handle2);
1029 igt_assert(ret == 0);
1031 /* after, no overlap */
1032 ret = gem_userptr(fd, ptr + PAGE_SIZE * 2, PAGE_SIZE, 0, &handle2);
1034 gem_close(fd, handle2);
1035 igt_assert(ret == 0);
1037 /* exactly overlapping */
1038 ret = gem_userptr(fd, ptr + PAGE_SIZE, PAGE_SIZE, 0, &handle2);
1040 gem_close(fd, handle2);
1041 igt_assert(ret == 0 || ret == expected);
1043 /* start overlaps */
1044 ret = gem_userptr(fd, ptr, PAGE_SIZE * 2, 0, &handle2);
1046 gem_close(fd, handle2);
1047 igt_assert(ret == 0 || ret == expected);
1050 ret = gem_userptr(fd, ptr + PAGE_SIZE, PAGE_SIZE * 2, 0, &handle2);
1052 gem_close(fd, handle2);
1053 igt_assert(ret == 0 || ret == expected);
1056 ret = gem_userptr(fd, ptr, PAGE_SIZE * 3, 0, &handle2);
1058 gem_close(fd, handle2);
1059 igt_assert(ret == 0 || ret == expected);
1061 gem_close(fd, handle);
1065 static void test_unmap(int fd, int expected)
1068 const unsigned int num_obj = 3;
1070 uint32_t bo[num_obj + 1];
1071 size_t map_size = sizeof(linear) * num_obj + (PAGE_SIZE - 1);
1074 ptr = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
1075 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
1076 igt_assert(ptr != MAP_FAILED);
1078 bo_ptr = (char *)ALIGN((unsigned long)ptr, PAGE_SIZE);
1080 for (i = 0; i < num_obj; i++, bo_ptr += sizeof(linear)) {
1081 ret = gem_userptr(fd, bo_ptr, sizeof(linear), 0, &bo[i]);
1082 igt_assert(ret == 0);
1085 bo[num_obj] = create_bo(fd, 0);
1087 for (i = 0; i < num_obj; i++)
1088 copy(fd, bo[num_obj], bo[i], 0);
1090 ret = munmap(ptr, map_size);
1091 igt_assert(ret == 0);
1093 for (i = 0; i < num_obj; i++)
1094 copy(fd, bo[num_obj], bo[i], expected);
1096 for (i = 0; i < (num_obj + 1); i++)
1097 gem_close(fd, bo[i]);
1100 static void test_unmap_after_close(int fd)
1103 const unsigned int num_obj = 3;
1105 uint32_t bo[num_obj + 1];
1106 size_t map_size = sizeof(linear) * num_obj + (PAGE_SIZE - 1);
1109 ptr = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
1110 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
1111 igt_assert(ptr != MAP_FAILED);
1113 bo_ptr = (char *)ALIGN((unsigned long)ptr, PAGE_SIZE);
1115 for (i = 0; i < num_obj; i++, bo_ptr += sizeof(linear)) {
1116 ret = gem_userptr(fd, bo_ptr, sizeof(linear), 0, &bo[i]);
1117 igt_assert(ret == 0);
1120 bo[num_obj] = create_bo(fd, 0);
1122 for (i = 0; i < num_obj; i++)
1123 copy(fd, bo[num_obj], bo[i], 0);
1125 for (i = 0; i < (num_obj + 1); i++)
1126 gem_close(fd, bo[i]);
1128 ret = munmap(ptr, map_size);
1129 igt_assert(ret == 0);
1132 static void test_unmap_cycles(int fd, int expected)
1136 for (i = 0; i < 1000; i++)
1137 test_unmap(fd, expected);
1140 struct stress_thread_data {
1145 static void *mm_stress_thread(void *data)
1147 struct stress_thread_data *stdata = (struct stress_thread_data *)data;
1151 while (!stdata->stop) {
1152 ptr = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE,
1153 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
1154 if (ptr == MAP_FAILED) {
1155 stdata->exit_code = -EFAULT;
1158 ret = munmap(ptr, PAGE_SIZE);
1160 stdata->exit_code = errno;
1168 static void test_stress_mm(int fd)
1172 unsigned int loops = 100000;
1175 struct stress_thread_data stdata;
1177 memset(&stdata, 0, sizeof(stdata));
1179 igt_assert(posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE) == 0);
1181 ret = pthread_create(&t, NULL, mm_stress_thread, &stdata);
1182 igt_assert(ret == 0);
1185 ret = gem_userptr(fd, ptr, PAGE_SIZE, 0, &handle);
1186 igt_assert(ret == 0);
1188 gem_close(fd, handle);
1194 ret = pthread_join(t, NULL);
1195 igt_assert(ret == 0);
1197 igt_assert(stdata.exit_code == 0);
1200 unsigned int total_ram;
1201 uint64_t aperture_size;
1205 int main(int argc, char **argv)
1207 int size = sizeof(linear);
1209 igt_skip_on_simulation();
1211 igt_subtest_init(argc, argv);
1216 fd = drm_open_any();
1217 igt_assert(fd >= 0);
1219 ret = has_userptr(fd);
1220 igt_skip_on_f(ret == 0, "No userptr support - %s (%d)\n",
1221 strerror(errno), ret);
1223 size = sizeof(linear);
1225 aperture_size = gem_aperture_size(fd);
1226 igt_info("Aperture size is %lu MiB\n", (long)(aperture_size / (1024*1024)));
1229 count = atoi(argv[1]);
1231 count = 2 * aperture_size / (1024*1024) / 3;
1233 total_ram = intel_get_total_ram_mb();
1234 igt_info("Total RAM is %u MiB\n", total_ram);
1236 if (count > total_ram * 3 / 4) {
1237 count = intel_get_total_ram_mb() * 3 / 4;
1238 igt_info("Not enough RAM to run test, reducing buffer count.\n");
1242 igt_subtest("input-checking")
1243 test_input_checking(fd);
1245 igt_subtest("usage-restrictions")
1246 test_usage_restrictions(fd);
1248 igt_subtest("invalid-null-pointer")
1249 test_invalid_null_pointer(fd);
1251 igt_subtest("invalid-gtt-mapping")
1252 test_invalid_gtt_mapping(fd);
1254 igt_subtest("forked-access")
1255 test_forked_access(fd);
1257 igt_subtest("forbidden-operations")
1258 test_forbidden_ops(fd);
1260 igt_info("Testing unsynchronized mappings...\n");
1261 gem_userptr_test_unsynchronized();
1263 igt_subtest("create-destroy-unsync")
1264 test_create_destroy(fd);
1266 igt_subtest("unsync-overlap")
1267 test_overlap(fd, 0);
1269 igt_subtest("unsync-unmap")
1272 igt_subtest("unsync-unmap-cycles")
1273 test_unmap_cycles(fd, 0);
1275 igt_subtest("unsync-unmap-after-close")
1276 test_unmap_after_close(fd);
1278 igt_subtest("coherency-unsync")
1279 test_coherency(fd, count);
1281 igt_subtest("dmabuf-unsync")
1284 for (unsigned flags = 0; flags < ALL_FORKING_EVICTIONS + 1; flags++) {
1285 igt_subtest_f("forked-unsync%s%s%s-%s",
1286 flags & FORKING_EVICTIONS_SWAPPING ? "-swapping" : "",
1287 flags & FORKING_EVICTIONS_DUP_DRMFD ? "-multifd" : "",
1288 flags & FORKING_EVICTIONS_MEMORY_PRESSURE ?
1289 "-mempressure" : "",
1290 flags & FORKING_EVICTIONS_INTERRUPTIBLE ?
1291 "interruptible" : "normal") {
1292 test_forking_evictions(fd, size, count, flags);
1296 igt_subtest("swapping-unsync-normal")
1297 test_swapping_evictions(fd, size, count);
1299 igt_subtest("minor-unsync-normal")
1300 test_minor_evictions(fd, size, count);
1302 igt_subtest("major-unsync-normal") {
1303 size = 200 * 1024 * 1024;
1304 count = (gem_aperture_size(fd) / size) + 2;
1305 test_major_evictions(fd, size, count);
1309 size = sizeof(linear);
1310 count = 2 * gem_aperture_size(fd) / (1024*1024) / 3;
1311 if (count > total_ram * 3 / 4)
1312 count = intel_get_total_ram_mb() * 3 / 4;
1315 igt_fork_signal_helper();
1317 igt_subtest("swapping-unsync-interruptible")
1318 test_swapping_evictions(fd, size, count);
1320 igt_subtest("minor-unsync-interruptible")
1321 test_minor_evictions(fd, size, count);
1323 igt_subtest("major-unsync-interruptible") {
1324 size = 200 * 1024 * 1024;
1325 count = (gem_aperture_size(fd) / size) + 2;
1326 test_major_evictions(fd, size, count);
1329 igt_stop_signal_helper();
1331 igt_info("Testing synchronized mappings...\n");
1334 size = sizeof(linear);
1335 count = 2 * gem_aperture_size(fd) / (1024*1024) / 3;
1336 if (count > total_ram * 3 / 4)
1337 count = intel_get_total_ram_mb() * 3 / 4;
1340 gem_userptr_test_synchronized();
1342 igt_subtest("process-exit")
1343 test_process_exit(fd, 0);
1345 igt_subtest("process-exit-gtt")
1346 test_process_exit(fd, PE_GTT_MAP);
1348 igt_subtest("process-exit-busy")
1349 test_process_exit(fd, PE_BUSY);
1351 igt_subtest("process-exit-gtt-busy")
1352 test_process_exit(fd, PE_GTT_MAP | PE_BUSY);
1354 igt_subtest("create-destroy-sync")
1355 test_create_destroy(fd);
1357 igt_subtest("sync-overlap")
1358 test_overlap(fd, EINVAL);
1360 igt_subtest("sync-unmap")
1361 test_unmap(fd, EFAULT);
1363 igt_subtest("sync-unmap-cycles")
1364 test_unmap_cycles(fd, EFAULT);
1366 igt_subtest("sync-unmap-after-close")
1367 test_unmap_after_close(fd);
1369 igt_subtest("stress-mm")
1372 igt_subtest("coherency-sync")
1373 test_coherency(fd, count);
1375 igt_subtest("dmabuf-sync")
1378 for (unsigned flags = 0; flags < ALL_FORKING_EVICTIONS + 1; flags++) {
1379 igt_subtest_f("forked-sync%s%s%s-%s",
1380 flags & FORKING_EVICTIONS_SWAPPING ? "-swapping" : "",
1381 flags & FORKING_EVICTIONS_DUP_DRMFD ? "-multifd" : "",
1382 flags & FORKING_EVICTIONS_MEMORY_PRESSURE ?
1383 "-mempressure" : "",
1384 flags & FORKING_EVICTIONS_INTERRUPTIBLE ?
1385 "interruptible" : "normal") {
1386 test_forking_evictions(fd, size, count, flags);
1390 igt_subtest("swapping-normal-sync")
1391 test_swapping_evictions(fd, size, count);
1393 igt_subtest("minor-normal-sync")
1394 test_minor_evictions(fd, size, count);
1396 igt_subtest("major-normal-sync") {
1397 size = 200 * 1024 * 1024;
1398 count = (gem_aperture_size(fd) / size) + 2;
1399 test_major_evictions(fd, size, count);
1404 count = 2 * gem_aperture_size(fd) / (1024*1024) / 3;
1405 if (count > total_ram * 3 / 4)
1406 count = intel_get_total_ram_mb() * 3 / 4;
1409 igt_fork_signal_helper();
1411 igt_subtest("swapping-sync-interruptible")
1412 test_swapping_evictions(fd, size, count);
1414 igt_subtest("minor-sync-interruptible")
1415 test_minor_evictions(fd, size, count);
1417 igt_subtest("major-sync-interruptible") {
1418 size = 200 * 1024 * 1024;
1419 count = (gem_aperture_size(fd) / size) + 2;
1420 test_major_evictions(fd, size, count);
1423 igt_stop_signal_helper();
1425 igt_subtest("access-control")
1426 test_access_control(fd);