2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #ifndef __I915_VMA_H__
26 #define __I915_VMA_H__
28 #include <linux/io-mapping.h>
29 #include <linux/rbtree.h>
31 #include <drm/drm_mm.h>
33 #include "gt/intel_ggtt_fencing.h"
34 #include "gem/i915_gem_object.h"
36 #include "i915_gem_gtt.h"
38 #include "i915_active.h"
39 #include "i915_request.h"
40 #include "i915_vma_resource.h"
41 #include "i915_vma_types.h"
44 i915_vma_instance(struct drm_i915_gem_object *obj,
45 struct i915_address_space *vm,
46 const struct i915_ggtt_view *view);
48 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
49 #define I915_VMA_RELEASE_MAP BIT(0)
51 static inline bool i915_vma_is_active(const struct i915_vma *vma)
53 return !i915_active_is_idle(&vma->active);
56 /* do not reserve memory to prevent deadlocks */
57 #define __EXEC_OBJECT_NO_RESERVE BIT(31)
59 int __must_check _i915_vma_move_to_active(struct i915_vma *vma,
60 struct i915_request *rq,
61 struct dma_fence *fence,
63 static inline int __must_check
64 i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq,
67 return _i915_vma_move_to_active(vma, rq, &rq->fence, flags);
70 #define __i915_vma_flags(v) ((unsigned long *)&(v)->flags.counter)
72 static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
74 return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
77 static inline bool i915_vma_is_dpt(const struct i915_vma *vma)
79 return i915_is_dpt(vma->vm);
82 static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
84 return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));
87 static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
89 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
90 set_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));
93 static inline bool i915_vma_unset_ggtt_write(struct i915_vma *vma)
95 return test_and_clear_bit(I915_VMA_GGTT_WRITE_BIT,
96 __i915_vma_flags(vma));
99 void i915_vma_flush_writes(struct i915_vma *vma);
101 static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
103 return test_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
106 static inline bool i915_vma_set_userfault(struct i915_vma *vma)
108 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
109 return test_and_set_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));
112 static inline void i915_vma_unset_userfault(struct i915_vma *vma)
114 return clear_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));
117 static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
119 return test_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));
122 static inline bool i915_vma_is_closed(const struct i915_vma *vma)
124 return !list_empty(&vma->closed_link);
127 static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
129 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
130 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
131 GEM_BUG_ON(upper_32_bits(vma->node.start));
132 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
133 return lower_32_bits(vma->node.start);
136 static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)
138 return i915_vm_to_ggtt(vma->vm)->pin_bias;
141 static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
143 i915_gem_object_get(vma->obj);
147 static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma)
149 if (likely(kref_get_unless_zero(&vma->obj->base.refcount)))
155 static inline void i915_vma_put(struct i915_vma *vma)
157 i915_gem_object_put(vma->obj);
161 i915_vma_compare(struct i915_vma *vma,
162 struct i915_address_space *vm,
163 const struct i915_ggtt_view *view)
167 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
169 cmp = ptrdiff(vma->vm, vm);
173 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0);
174 cmp = vma->ggtt_view.type;
182 assert_i915_gem_gtt_types();
184 /* ggtt_view.type also encodes its size so that we both distinguish
185 * different views using it as a "type" and also use a compact (no
186 * accessing of uninitialised padding bytes) memcmp without storing
187 * an extra parameter or adding more code.
189 * To ensure that the memcmp is valid for all branches of the union,
190 * even though the code looks like it is just comparing one branch,
191 * we assert above that all branches have the same address, and that
192 * each branch has a unique type/size.
194 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL);
195 BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED);
196 BUILD_BUG_ON(I915_GGTT_VIEW_ROTATED >= I915_GGTT_VIEW_REMAPPED);
197 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
198 offsetof(typeof(*view), partial));
199 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
200 offsetof(typeof(*view), remapped));
201 return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
204 struct i915_vma_work *i915_vma_work(void);
205 int i915_vma_bind(struct i915_vma *vma,
206 enum i915_cache_level cache_level,
208 struct i915_vma_work *work,
209 struct i915_vma_resource *vma_res);
211 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color);
212 bool i915_vma_misplaced(const struct i915_vma *vma,
213 u64 size, u64 alignment, u64 flags);
214 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
215 void i915_vma_revoke_mmap(struct i915_vma *vma);
216 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);
217 int __i915_vma_unbind(struct i915_vma *vma);
218 int __must_check i915_vma_unbind(struct i915_vma *vma);
219 int __must_check i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm);
220 int __must_check i915_vma_unbind_unlocked(struct i915_vma *vma);
221 void i915_vma_unlink_ctx(struct i915_vma *vma);
222 void i915_vma_close(struct i915_vma *vma);
223 void i915_vma_reopen(struct i915_vma *vma);
225 void i915_vma_destroy_locked(struct i915_vma *vma);
226 void i915_vma_destroy(struct i915_vma *vma);
228 #define assert_vma_held(vma) dma_resv_assert_held((vma)->obj->base.resv)
230 static inline void i915_vma_lock(struct i915_vma *vma)
232 dma_resv_lock(vma->obj->base.resv, NULL);
235 static inline void i915_vma_unlock(struct i915_vma *vma)
237 dma_resv_unlock(vma->obj->base.resv);
241 i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
242 u64 size, u64 alignment, u64 flags);
244 static inline int __must_check
245 i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
247 struct i915_gem_ww_ctx ww;
250 i915_gem_ww_ctx_init(&ww, true);
252 err = i915_gem_object_lock(vma->obj, &ww);
254 err = i915_vma_pin_ww(vma, &ww, size, alignment, flags);
255 if (err == -EDEADLK) {
256 err = i915_gem_ww_ctx_backoff(&ww);
260 i915_gem_ww_ctx_fini(&ww);
265 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
266 u32 align, unsigned int flags);
268 static inline int i915_vma_pin_count(const struct i915_vma *vma)
270 return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;
273 static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
275 return i915_vma_pin_count(vma);
278 static inline void __i915_vma_pin(struct i915_vma *vma)
280 atomic_inc(&vma->flags);
281 GEM_BUG_ON(!i915_vma_is_pinned(vma));
284 static inline void __i915_vma_unpin(struct i915_vma *vma)
286 GEM_BUG_ON(!i915_vma_is_pinned(vma));
287 atomic_dec(&vma->flags);
290 static inline void i915_vma_unpin(struct i915_vma *vma)
292 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
293 __i915_vma_unpin(vma);
296 static inline bool i915_vma_is_bound(const struct i915_vma *vma,
299 return atomic_read(&vma->flags) & where;
302 static inline bool i915_node_color_differs(const struct drm_mm_node *node,
305 return drm_mm_node_allocated(node) && node->color != color;
309 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
312 * The passed in VMA has to be pinned in the global GTT mappable region.
313 * An extra pinning of the VMA is acquired for the return iomapping,
314 * the caller must call i915_vma_unpin_iomap to relinquish the pinning
315 * after the iomapping is no longer required.
317 * Returns a valid iomapped pointer or ERR_PTR.
319 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
322 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
325 * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
327 * This function is only valid to be called on a VMA previously
328 * iomapped by the caller with i915_vma_pin_iomap().
330 void i915_vma_unpin_iomap(struct i915_vma *vma);
333 * i915_vma_pin_fence - pin fencing state
334 * @vma: vma to pin fencing for
336 * This pins the fencing state (whether tiled or untiled) to make sure the
337 * vma (and its object) is ready to be used as a scanout target. Fencing
338 * status must be synchronize first by calling i915_vma_get_fence():
340 * The resulting fence pin reference must be released again with
341 * i915_vma_unpin_fence().
345 * True if the vma has a fence, false otherwise.
347 int __must_check i915_vma_pin_fence(struct i915_vma *vma);
348 void i915_vma_revoke_fence(struct i915_vma *vma);
350 int __i915_vma_pin_fence(struct i915_vma *vma);
352 static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
354 GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0);
355 atomic_dec(&vma->fence->pin_count);
359 * i915_vma_unpin_fence - unpin fencing state
360 * @vma: vma to unpin fencing for
362 * This releases the fence pin reference acquired through
363 * i915_vma_pin_fence. It will handle both objects with and without an
364 * attached fence correctly, callers do not need to distinguish this.
367 i915_vma_unpin_fence(struct i915_vma *vma)
370 __i915_vma_unpin_fence(vma);
373 void i915_vma_parked(struct intel_gt *gt);
375 static inline bool i915_vma_is_scanout(const struct i915_vma *vma)
377 return test_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));
380 static inline void i915_vma_mark_scanout(struct i915_vma *vma)
382 set_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));
385 static inline void i915_vma_clear_scanout(struct i915_vma *vma)
387 clear_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));
390 #define for_each_until(cond) if (cond) break; else
393 * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object.
394 * @V: the #i915_vma iterator
395 * @OBJ: the #drm_i915_gem_object
397 * GGTT VMA are placed at the being of the object's vma_list, see
398 * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA,
399 * or the list is empty ofc.
401 #define for_each_ggtt_vma(V, OBJ) \
402 list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \
403 for_each_until(!i915_vma_is_ggtt(V))
405 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma);
406 void i915_vma_make_shrinkable(struct i915_vma *vma);
407 void i915_vma_make_purgeable(struct i915_vma *vma);
409 int i915_vma_wait_for_bind(struct i915_vma *vma);
411 static inline int i915_vma_sync(struct i915_vma *vma)
413 /* Wait for the asynchronous bindings and pending GPU reads */
414 return i915_active_wait(&vma->active);
418 * i915_vma_get_current_resource - Get the current resource of the vma
419 * @vma: The vma to get the current resource from.
421 * It's illegal to call this function if the vma is not bound.
423 * Return: A refcounted pointer to the current vma resource
424 * of the vma, assuming the vma is bound.
426 static inline struct i915_vma_resource *
427 i915_vma_get_current_resource(struct i915_vma *vma)
429 return i915_vma_resource_get(vma->resource);
432 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
433 void i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
434 struct i915_vma *vma);
437 void i915_vma_module_exit(void);
438 int i915_vma_module_init(void);
440 I915_SELFTEST_DECLARE(int i915_vma_get_pages(struct i915_vma *vma));
441 I915_SELFTEST_DECLARE(void i915_vma_put_pages(struct i915_vma *vma));