1 // SPDX-License-Identifier: GPL-2.0+
3 * Copyright (C) 2018, Tuomas Tynkkynen <tuomas.tynkkynen@iki.fi>
4 * Copyright (C) 2018, Bin Meng <bmeng.cn@gmail.com>
6 * virtio ring implementation
14 #include <virtio_types.h>
16 #include <virtio_ring.h>
17 #include <linux/bug.h>
18 #include <linux/compat.h>
19 #include <linux/kernel.h>
21 static void *virtio_alloc_pages(struct udevice *vdev, u32 npages)
23 return memalign(PAGE_SIZE, npages * PAGE_SIZE);
26 static void virtio_free_pages(struct udevice *vdev, void *ptr, u32 npages)
31 static int __bb_force_page_align(struct bounce_buffer *state)
33 const ulong align_mask = PAGE_SIZE - 1;
35 if ((ulong)state->user_buffer & align_mask)
38 if (state->len != state->len_aligned)
44 static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i,
45 struct virtio_sg *sg, u16 flags)
47 struct vring_desc_shadow *desc_shadow = &vq->vring_desc_shadow[i];
48 struct vring_desc *desc = &vq->vring.desc[i];
51 if (IS_ENABLED(CONFIG_BOUNCE_BUFFER) && vq->vring.bouncebufs) {
52 struct bounce_buffer *bb = &vq->vring.bouncebufs[i];
56 if (flags & VRING_DESC_F_WRITE)
57 bbflags = GEN_BB_WRITE;
59 bbflags = GEN_BB_READ;
61 ret = bounce_buffer_start_extalign(bb, sg->addr, sg->length,
63 __bb_force_page_align);
65 debug("%s: failed to allocate bounce buffer (length 0x%zx)\n",
66 vq->vdev->name, sg->length);
69 addr = bb->bounce_buffer;
74 /* Update the shadow descriptor. */
75 desc_shadow->addr = (u64)(uintptr_t)addr;
76 desc_shadow->len = sg->length;
77 desc_shadow->flags = flags;
79 /* Update the shared descriptor to match the shadow. */
80 desc->addr = cpu_to_virtio64(vq->vdev, desc_shadow->addr);
81 desc->len = cpu_to_virtio32(vq->vdev, desc_shadow->len);
82 desc->flags = cpu_to_virtio16(vq->vdev, desc_shadow->flags);
83 desc->next = cpu_to_virtio16(vq->vdev, desc_shadow->next);
85 return desc_shadow->next;
88 static void virtqueue_detach_desc(struct virtqueue *vq, unsigned int idx)
90 struct vring_desc *desc = &vq->vring.desc[idx];
91 struct bounce_buffer *bb;
93 if (!IS_ENABLED(CONFIG_BOUNCE_BUFFER) || !vq->vring.bouncebufs)
96 bb = &vq->vring.bouncebufs[idx];
97 bounce_buffer_stop(bb);
98 desc->addr = cpu_to_virtio64(vq->vdev, (u64)(uintptr_t)bb->user_buffer);
101 int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[],
102 unsigned int out_sgs, unsigned int in_sgs)
104 struct vring_desc *desc;
105 unsigned int descs_used = out_sgs + in_sgs;
106 unsigned int i, n, avail, uninitialized_var(prev);
109 WARN_ON(descs_used == 0);
111 head = vq->free_head;
113 desc = vq->vring.desc;
116 if (vq->num_free < descs_used) {
117 debug("Can't add buf len %i - avail = %i\n",
118 descs_used, vq->num_free);
120 * FIXME: for historical reasons, we force a notify here if
121 * there are outgoing parts to the buffer. Presumably the
122 * host should service the ring ASAP.
125 virtio_notify(vq->vdev, vq);
129 for (n = 0; n < descs_used; n++) {
130 u16 flags = VRING_DESC_F_NEXT;
133 flags |= VRING_DESC_F_WRITE;
135 i = virtqueue_attach_desc(vq, i, sgs[n], flags);
137 /* Last one doesn't continue */
138 vq->vring_desc_shadow[prev].flags &= ~VRING_DESC_F_NEXT;
139 desc[prev].flags = cpu_to_virtio16(vq->vdev, vq->vring_desc_shadow[prev].flags);
141 /* We're using some buffers from the free list. */
142 vq->num_free -= descs_used;
144 /* Update free pointer */
147 /* Mark the descriptor as the head of a chain. */
148 vq->vring_desc_shadow[head].chain_head = true;
151 * Put entry in available array (but don't update avail->idx
152 * until they do sync).
154 avail = vq->avail_idx_shadow & (vq->vring.num - 1);
155 vq->vring.avail->ring[avail] = cpu_to_virtio16(vq->vdev, head);
158 * Descriptors and available array need to be set before we expose the
159 * new available array entries.
162 vq->avail_idx_shadow++;
163 vq->vring.avail->idx = cpu_to_virtio16(vq->vdev, vq->avail_idx_shadow);
167 * This is very unlikely, but theoretically possible.
170 if (unlikely(vq->num_added == (1 << 16) - 1))
176 static bool virtqueue_kick_prepare(struct virtqueue *vq)
182 * We need to expose available array entries before checking
187 old = vq->avail_idx_shadow - vq->num_added;
188 new = vq->avail_idx_shadow;
192 needs_kick = vring_need_event(virtio16_to_cpu(vq->vdev,
193 vring_avail_event(&vq->vring)), new, old);
195 needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(vq->vdev,
196 VRING_USED_F_NO_NOTIFY));
202 void virtqueue_kick(struct virtqueue *vq)
204 if (virtqueue_kick_prepare(vq))
205 virtio_notify(vq->vdev, vq);
208 static void detach_buf(struct virtqueue *vq, unsigned int head)
212 /* Unmark the descriptor as the head of a chain. */
213 vq->vring_desc_shadow[head].chain_head = false;
215 /* Put back on free list: unmap first-level descriptors and find end */
218 while (vq->vring_desc_shadow[i].flags & VRING_DESC_F_NEXT) {
219 virtqueue_detach_desc(vq, i);
220 i = vq->vring_desc_shadow[i].next;
224 virtqueue_detach_desc(vq, i);
225 vq->vring_desc_shadow[i].next = vq->free_head;
226 vq->free_head = head;
228 /* Plus final descriptor */
232 static inline bool more_used(const struct virtqueue *vq)
234 return vq->last_used_idx != virtio16_to_cpu(vq->vdev,
235 vq->vring.used->idx);
238 void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len)
243 if (!more_used(vq)) {
244 debug("(%s.%d): No more buffers in queue\n",
245 vq->vdev->name, vq->index);
249 /* Only get used array entries after they have been exposed by host */
252 last_used = (vq->last_used_idx & (vq->vring.num - 1));
253 i = virtio32_to_cpu(vq->vdev, vq->vring.used->ring[last_used].id);
255 *len = virtio32_to_cpu(vq->vdev,
256 vq->vring.used->ring[last_used].len);
257 debug("(%s.%d): last used idx %u with len %u\n",
258 vq->vdev->name, vq->index, i, *len);
261 if (unlikely(i >= vq->vring.num)) {
262 printf("(%s.%d): id %u out of range\n",
263 vq->vdev->name, vq->index, i);
267 if (unlikely(!vq->vring_desc_shadow[i].chain_head)) {
268 printf("(%s.%d): id %u is not a head\n",
269 vq->vdev->name, vq->index, i);
276 * If we expect an interrupt for the next entry, tell host
277 * by writing event index and flush out the write before
278 * the read in the next get_buf call.
280 if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
281 virtio_store_mb(&vring_used_event(&vq->vring),
282 cpu_to_virtio16(vq->vdev, vq->last_used_idx));
284 return (void *)(uintptr_t)vq->vring_desc_shadow[i].addr;
287 static struct virtqueue *__vring_new_virtqueue(unsigned int index,
289 struct udevice *udev)
292 struct virtqueue *vq;
293 struct vring_desc_shadow *vring_desc_shadow;
294 struct virtio_dev_priv *uc_priv = dev_get_uclass_priv(udev);
295 struct udevice *vdev = uc_priv->vdev;
297 vq = malloc(sizeof(*vq));
301 vring_desc_shadow = calloc(vring.num, sizeof(struct vring_desc_shadow));
302 if (!vring_desc_shadow) {
309 vq->num_free = vring.num;
311 vq->vring_desc_shadow = vring_desc_shadow;
312 vq->last_used_idx = 0;
313 vq->avail_flags_shadow = 0;
314 vq->avail_idx_shadow = 0;
316 list_add_tail(&vq->list, &uc_priv->vqs);
318 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
320 /* Tell other side not to bother us */
321 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
323 vq->vring.avail->flags = cpu_to_virtio16(vdev,
324 vq->avail_flags_shadow);
326 /* Put everything in free lists */
328 for (i = 0; i < vring.num - 1; i++)
329 vq->vring_desc_shadow[i].next = i + 1;
334 struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,
335 unsigned int vring_align,
336 struct udevice *udev)
338 struct virtio_dev_priv *uc_priv = dev_get_uclass_priv(udev);
339 struct udevice *vdev = uc_priv->vdev;
340 struct virtqueue *vq;
342 struct bounce_buffer *bbs = NULL;
345 /* We assume num is a power of 2 */
346 if (num & (num - 1)) {
347 printf("Bad virtqueue length %u\n", num);
351 /* TODO: allocate each queue chunk individually */
352 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
353 size_t sz = vring_size(num, vring_align);
355 queue = virtio_alloc_pages(vdev, DIV_ROUND_UP(sz, PAGE_SIZE));
364 /* Try to get a single page. You are my only hope! */
365 queue = virtio_alloc_pages(vdev, 1);
370 memset(queue, 0, vring_size(num, vring_align));
372 if (virtio_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
373 bbs = calloc(num, sizeof(*bbs));
378 vring_init(&vring, num, queue, vring_align, bbs);
380 vq = __vring_new_virtqueue(index, vring, udev);
384 debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name,
392 virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE));
396 void vring_del_virtqueue(struct virtqueue *vq)
398 virtio_free_pages(vq->vdev, vq->vring.desc,
399 DIV_ROUND_UP(vq->vring.size, PAGE_SIZE));
400 free(vq->vring_desc_shadow);
402 free(vq->vring.bouncebufs);
406 unsigned int virtqueue_get_vring_size(struct virtqueue *vq)
408 return vq->vring.num;
411 ulong virtqueue_get_desc_addr(struct virtqueue *vq)
413 return (ulong)vq->vring.desc;
416 ulong virtqueue_get_avail_addr(struct virtqueue *vq)
418 return (ulong)vq->vring.desc +
419 ((char *)vq->vring.avail - (char *)vq->vring.desc);
422 ulong virtqueue_get_used_addr(struct virtqueue *vq)
424 return (ulong)vq->vring.desc +
425 ((char *)vq->vring.used - (char *)vq->vring.desc);
428 bool virtqueue_poll(struct virtqueue *vq, u16 last_used_idx)
432 return last_used_idx != virtio16_to_cpu(vq->vdev, vq->vring.used->idx);
435 void virtqueue_dump(struct virtqueue *vq)
439 printf("virtqueue %p for dev %s:\n", vq, vq->vdev->name);
440 printf("\tindex %u, phys addr %p num %u\n",
441 vq->index, vq->vring.desc, vq->vring.num);
442 printf("\tfree_head %u, num_added %u, num_free %u\n",
443 vq->free_head, vq->num_added, vq->num_free);
444 printf("\tlast_used_idx %u, avail_flags_shadow %u, avail_idx_shadow %u\n",
445 vq->last_used_idx, vq->avail_flags_shadow, vq->avail_idx_shadow);
447 printf("Shadow descriptor dump:\n");
448 for (i = 0; i < vq->vring.num; i++) {
449 struct vring_desc_shadow *desc = &vq->vring_desc_shadow[i];
451 printf("\tdesc_shadow[%u] = { 0x%llx, len %u, flags %u, next %u }\n",
452 i, desc->addr, desc->len, desc->flags, desc->next);
455 printf("Avail ring dump:\n");
456 printf("\tflags %u, idx %u\n",
457 vq->vring.avail->flags, vq->vring.avail->idx);
458 for (i = 0; i < vq->vring.num; i++) {
459 printf("\tavail[%u] = %u\n",
460 i, vq->vring.avail->ring[i]);
463 printf("Used ring dump:\n");
464 printf("\tflags %u, idx %u\n",
465 vq->vring.used->flags, vq->vring.used->idx);
466 for (i = 0; i < vq->vring.num; i++) {
467 printf("\tused[%u] = { %u, %u }\n", i,
468 vq->vring.used->ring[i].id, vq->vring.used->ring[i].len);