drivers/dma/idxd/dma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/io-64-nonatomic-lo-hi.h>
   9 #include <linux/dmaengine.h>
  10 #include <uapi/linux/idxd.h>
  11 #include "../dmaengine.h"
  12 #include "registers.h"
  13 #include "idxd.h"
  14
  15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
  16 {
  17         return container_of(c, struct idxd_wq, dma_chan);
  18 }
  19
  20 void idxd_dma_complete_txd(struct idxd_desc *desc,
  21                            enum idxd_complete_type comp_type)
  22 {
  23         struct dma_async_tx_descriptor *tx;
  24         struct dmaengine_result res;
  25         int complete = 1;
  26
  27         if (desc->completion->status == DSA_COMP_SUCCESS)
  28                 res.result = DMA_TRANS_NOERROR;
  29         else if (desc->completion->status)
  30                 res.result = DMA_TRANS_WRITE_FAILED;
  31         else if (comp_type == IDXD_COMPLETE_ABORT)
  32                 res.result = DMA_TRANS_ABORTED;
  33         else
  34                 complete = 0;
  35
  36         tx = &desc->txd;
  37         if (complete && tx->cookie) {
  38                 dma_cookie_complete(tx);
  39                 dma_descriptor_unmap(tx);
  40                 dmaengine_desc_get_callback_invoke(tx, &res);
  41                 tx->callback = NULL;
  42                 tx->callback_result = NULL;
  43         }
  44 }
  45
  46 static void op_flag_setup(unsigned long flags, u32 *desc_flags)
  47 {
  48         *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
  49         if (flags & DMA_PREP_INTERRUPT)
  50                 *desc_flags |= IDXD_OP_FLAG_RCI;
  51 }
  52
  53 static inline void set_completion_address(struct idxd_desc *desc,
  54                                           u64 *compl_addr)
  55 {
  56                 *compl_addr = desc->compl_dma;
  57 }
  58
  59 static inline void idxd_prep_desc_common(struct idxd_wq *wq,
  60                                          struct dsa_hw_desc *hw, char opcode,
  61                                          u64 addr_f1, u64 addr_f2, u64 len,
  62                                          u64 compl, u32 flags)
  63 {
  64         hw->flags = flags;
  65         hw->opcode = opcode;
  66         hw->src_addr = addr_f1;
  67         hw->dst_addr = addr_f2;
  68         hw->xfer_size = len;
  69         hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
  70         hw->completion_addr = compl;
  71 }
  72
  73 static struct dma_async_tx_descriptor *
  74 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
  75                        dma_addr_t dma_src, size_t len, unsigned long flags)
  76 {
  77         struct idxd_wq *wq = to_idxd_wq(c);
  78         u32 desc_flags;
  79         struct idxd_device *idxd = wq->idxd;
  80         struct idxd_desc *desc;
  81
  82         if (wq->state != IDXD_WQ_ENABLED)
  83                 return NULL;
  84
  85         if (len > idxd->max_xfer_bytes)
  86                 return NULL;
  87
  88         op_flag_setup(flags, &desc_flags);
  89         desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
  90         if (IS_ERR(desc))
  91                 return NULL;
  92
  93         idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
  94                               dma_src, dma_dest, len, desc->compl_dma,
  95                               desc_flags);
  96
  97         desc->txd.flags = flags;
  98
  99         return &desc->txd;
 100 }
 101
 102 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
 103 {
 104         struct idxd_wq *wq = to_idxd_wq(chan);
 105         struct device *dev = &wq->idxd->pdev->dev;
 106
 107         idxd_wq_get(wq);
 108         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 109                 idxd_wq_refcount(wq));
 110         return 0;
 111 }
 112
 113 static void idxd_dma_free_chan_resources(struct dma_chan *chan)
 114 {
 115         struct idxd_wq *wq = to_idxd_wq(chan);
 116         struct device *dev = &wq->idxd->pdev->dev;
 117
 118         idxd_wq_put(wq);
 119         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 120                 idxd_wq_refcount(wq));
 121 }
 122
 123 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
 124                                           dma_cookie_t cookie,
 125                                           struct dma_tx_state *txstate)
 126 {
 127         return DMA_OUT_OF_ORDER;
 128 }
 129
 130 /*
 131  * issue_pending() does not need to do anything since tx_submit() does the job
 132  * already.
 133  */
 134 static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
 135 {
 136 }
 137
 138 dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 139 {
 140         struct dma_chan *c = tx->chan;
 141         struct idxd_wq *wq = to_idxd_wq(c);
 142         dma_cookie_t cookie;
 143         int rc;
 144         struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
 145
 146         cookie = dma_cookie_assign(tx);
 147
 148         rc = idxd_submit_desc(wq, desc);
 149         if (rc < 0) {
 150                 idxd_free_desc(wq, desc);
 151                 return rc;
 152         }
 153
 154         return cookie;
 155 }
 156
 157 static void idxd_dma_release(struct dma_device *device)
 158 {
 159 }
 160
 161 int idxd_register_dma_device(struct idxd_device *idxd)
 162 {
 163         struct dma_device *dma = &idxd->dma_dev;
 164
 165         INIT_LIST_HEAD(&dma->channels);
 166         dma->dev = &idxd->pdev->dev;
 167
 168         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 169         dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
 170         dma->device_release = idxd_dma_release;
 171
 172         if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
 173                 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 174                 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
 175         }
 176
 177         dma->device_tx_status = idxd_dma_tx_status;
 178         dma->device_issue_pending = idxd_dma_issue_pending;
 179         dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
 180         dma->device_free_chan_resources = idxd_dma_free_chan_resources;
 181
 182         return dma_async_device_register(&idxd->dma_dev);
 183 }
 184
 185 void idxd_unregister_dma_device(struct idxd_device *idxd)
 186 {
 187         dma_async_device_unregister(&idxd->dma_dev);
 188 }
 189
 190 int idxd_register_dma_channel(struct idxd_wq *wq)
 191 {
 192         struct idxd_device *idxd = wq->idxd;
 193         struct dma_device *dma = &idxd->dma_dev;
 194         struct dma_chan *chan = &wq->dma_chan;
 195         int rc;
 196
 197         memset(&wq->dma_chan, 0, sizeof(struct dma_chan));
 198         chan->device = dma;
 199         list_add_tail(&chan->device_node, &dma->channels);
 200         rc = dma_async_device_channel_register(dma, chan);
 201         if (rc < 0)
 202                 return rc;
 203
 204         return 0;
 205 }
 206
 207 void idxd_unregister_dma_channel(struct idxd_wq *wq)
 208 {
 209         struct dma_chan *chan = &wq->dma_chan;
 210
 211         dma_async_device_channel_unregister(&wq->idxd->dma_dev, chan);
 212         list_del(&chan->device_node);
 213 }