#include "nouveau_dma.h"
#include "nouveau_local.h"
-static __inline__ uint32_t
+static inline uint32_t
READ_GET(struct nouveau_channel_priv *nvchan)
{
- return ((*nvchan->get - nvchan->dma.base) >> 2);
+ return *nvchan->get;
}
-static __inline__ void
+static inline void
WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val)
{
- uint32_t put = ((val << 2) + nvchan->dma.base);
+ uint32_t put = ((val << 2) + nvchan->dma->base);
volatile int dum;
NOUVEAU_DMA_BARRIER;
dum = READ_GET(nvchan);
*nvchan->put = put;
- nvchan->dma.put = val;
+ nvchan->dma->put = val;
#ifdef NOUVEAU_DMA_TRACE
NOUVEAU_MSG("WRITE_PUT %d/0x%08x\n", nvchan->drm.channel, put);
#endif
NOUVEAU_DMA_BARRIER;
}
+static inline int
+LOCAL_GET(struct nouveau_dma_priv *dma, uint32_t *val)
+{
+ uint32_t get = *val;
+
+ if (get >= dma->base && get <= (dma->base + (dma->max << 2))) {
+ *val = (get - dma->base) >> 2;
+ return 1;
+ }
+
+ return 0;
+}
+
void
nouveau_dma_channel_init(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
int i;
- nvchan->dma.base = nvchan->drm.put_base;
- nvchan->dma.cur = nvchan->dma.put = 0;
- nvchan->dma.max = (nvchan->drm.cmdbuf_size >> 2) - 2;
- nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
+ nvchan->dma = &nvchan->dma_master;
+ nvchan->dma->base = nvchan->drm.put_base;
+ nvchan->dma->cur = nvchan->dma->put = 0;
+ nvchan->dma->max = (nvchan->drm.cmdbuf_size >> 2) - 2;
+ nvchan->dma->free = nvchan->dma->max - nvchan->dma->cur;
RING_SPACE_CH(chan, RING_SKIPS);
for (i = 0; i < RING_SKIPS; i++)
return - EBUSY; \
} while(0)
-#define IN_MASTER_RING(chan, ptr) ((ptr) <= (chan)->dma.max)
-
int
nouveau_dma_wait(struct nouveau_channel *chan, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+ struct nouveau_dma_priv *dma = nvchan->dma;
uint32_t get, t_start;
FIRE_RING_CH(chan);
t_start = NOUVEAU_TIME_MSEC();
- while (nvchan->dma.free < size) {
+ while (dma->free < size) {
CHECK_TIMEOUT();
get = READ_GET(nvchan);
- if (!IN_MASTER_RING(nvchan, get))
+ if (!LOCAL_GET(dma, &get))
continue;
- if (nvchan->dma.put >= get) {
- nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
+ if (dma->put >= get) {
+ dma->free = dma->max - dma->cur;
- if (nvchan->dma.free < size) {
+ if (dma->free < size) {
#ifdef NOUVEAU_DMA_DEBUG
- nvchan->dma.push_free = 1;
+ dma->push_free = 1;
#endif
- OUT_RING_CH(chan,
- 0x20000000 | nvchan->dma.base);
+ OUT_RING_CH(chan, 0x20000000 | dma->base);
if (get <= RING_SKIPS) {
/*corner case - will be idle*/
- if (nvchan->dma.put <= RING_SKIPS)
+ if (dma->put <= RING_SKIPS)
WRITE_PUT(nvchan,
RING_SKIPS + 1);
do {
CHECK_TIMEOUT();
get = READ_GET(nvchan);
- if (!IN_MASTER_RING(nvchan,
- get))
- continue;
+ if (!LOCAL_GET(dma, &get))
+ get = 0;
} while (get <= RING_SKIPS);
}
WRITE_PUT(nvchan, RING_SKIPS);
- nvchan->dma.cur = nvchan->dma.put = RING_SKIPS;
- nvchan->dma.free = get - (RING_SKIPS + 1);
+ dma->cur = dma->put = RING_SKIPS;
+ dma->free = get - (RING_SKIPS + 1);
}
} else {
- nvchan->dma.free = get - nvchan->dma.cur - 1;
+ dma->free = get - dma->cur - 1;
}
}
unsigned mthd_count = 0;
while (get != put) {
- uint32_t gpuget = (get << 2) + nvchan->dma.base;
+ uint32_t gpuget = (get << 2) + nvchan->drm.put_base;
uint32_t data;
if (get < 0 || get >= nvchan->drm.cmdbuf_size) {
nouveau_dma_kickoff(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+ struct nouveau_dma_priv *dma = nvchan->dma;
- if (nvchan->dma.cur == nvchan->dma.put)
+ if (dma->cur == dma->put)
return;
#ifdef NOUVEAU_DMA_DEBUG
- if (nvchan->dma.push_free) {
- NOUVEAU_ERR("Packet incomplete: %d left\n",
- nvchan->dma.push_free);
+ if (dma->push_free) {
+ NOUVEAU_ERR("Packet incomplete: %d left\n", dma->push_free);
return;
}
#endif
#ifdef NOUVEAU_DMA_DUMP_POSTRELOC_PUSHBUF
- nouveau_dma_parse_pushbuf(chan, nvchan->dma.put, nvchan->dma.cur);
+ nouveau_dma_parse_pushbuf(chan, dma->put, dma->cur);
#endif
- WRITE_PUT(nvchan, nvchan->dma.cur);
+ WRITE_PUT(nvchan, dma->cur);
}
nouveau_dma_out(struct nouveau_channel *chan, uint32_t data)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+ struct nouveau_dma_priv *dma = nvchan->dma;
#ifdef NOUVEAU_DMA_DEBUG
- if (nvchan->dma.push_free == 0) {
- NOUVEAU_ERR("No space left in packet. Error at %s\n",faulty);
+ if (dma->push_free == 0) {
+ NOUVEAU_ERR("No space left in packet at %s\n", faulty);
return;
}
- nvchan->dma.push_free--;
+ dma->push_free--;
#endif
#ifdef NOUVEAU_DMA_TRACE
{
- uint32_t offset = (nvchan->dma.cur << 2) + nvchan->dma.base;
+ uint32_t offset = (dma->cur << 2) + dma->base;
NOUVEAU_MSG("\tOUT_RING %d/0x%08x -> 0x%08x\n",
nvchan->drm.channel, offset, data);
}
#endif
- nvchan->pushbuf[nvchan->dma.cur++] = data;
+ nvchan->pushbuf[dma->cur + (dma->base - nvchan->drm.put_base)/4] = data;
+ dma->cur++;
}
static inline void
nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
- (void)chan;
+ struct nouveau_dma_priv *dma = nvchan->dma;
+ (void)dma;
#ifdef NOUVEAU_DMA_DEBUG
- if (nvchan->dma.push_free < size) {
+ if (dma->push_free < size) {
NOUVEAU_ERR("Packet too small. Free=%d, Need=%d\n",
- nvchan->dma.push_free, size);
+ dma->push_free, size);
return;
}
#endif
ptr++;
}
#else
- memcpy(&nvchan->pushbuf[nvchan->dma.cur], ptr, size << 2);
+ memcpy(&nvchan->pushbuf[dma->cur], ptr, size << 2);
#ifdef NOUVEAU_DMA_DEBUG
- nvchan->dma.push_free -= size;
+ dma->push_free -= size;
#endif
- nvchan->dma.cur += size;
+ dma->cur += size;
#endif
}
nouveau_dma_space(struct nouveau_channel *chan, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+ struct nouveau_dma_priv *dma = nvchan->dma;
- if (nvchan->dma.free < size) {
+ if (dma->free < size) {
if (nouveau_dma_wait(chan, size) && chan->hang_notify)
chan->hang_notify(chan);
}
- nvchan->dma.free -= size;
+ dma->free -= size;
#ifdef NOUVEAU_DMA_DEBUG
- nvchan->dma.push_free = size;
+ dma->push_free = size;
#endif
}
int method, int size, const char* file, int line)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
- (void)nvchan;
+ struct nouveau_dma_priv *dma = nvchan->dma;
+ (void)dma;
#ifdef NOUVEAU_DMA_TRACE
NOUVEAU_MSG("BEGIN_RING %d/%08x/%d/0x%04x/%d\n", nvchan->drm.channel,
#endif
#ifdef NOUVEAU_DMA_DEBUG
- if (nvchan->dma.push_free) {
- NOUVEAU_ERR("Previous packet incomplete: %d left. Error at %s\n",
- nvchan->dma.push_free,faulty);
+ if (dma->push_free) {
+ NOUVEAU_ERR("Previous packet incomplete: %d left at %s\n",
+ dma->push_free, faulty);
return;
}
sprintf(faulty,"%s:%d",file,line);
struct nouveau_pushbuf_priv {
struct nouveau_pushbuf base;
- struct nouveau_pushbuf *next;
- struct nouveau_resource *res;
- struct nouveau_fence *fence;
+ unsigned nop_jump;
+ unsigned start;
+ unsigned size;
uint64_t buffers;
int nr_buffers;
nouveau_pushbuf_init(struct nouveau_channel *);
extern int
-nouveau_pushbuf_flush(struct nouveau_channel *);
+nouveau_pushbuf_flush(struct nouveau_channel *, unsigned min);
extern int
nouveau_pushbuf_emit_reloc(struct nouveau_channel *, void *ptr,
struct nouveau_bo *, uint32_t data, uint32_t flags,
uint32_t vor, uint32_t tor);
+struct nouveau_dma_priv {
+ uint32_t base;
+ uint32_t max;
+ uint32_t cur;
+ uint32_t put;
+ uint32_t free;
+
+ int push_free;
+} dma;
+
struct nouveau_channel_priv {
struct nouveau_channel base;
volatile uint32_t *get;
volatile uint32_t *ref_cnt;
- struct {
- uint32_t base, max;
- uint32_t cur, put;
- uint32_t free;
-
- int push_free;
- } dma;
+ struct nouveau_dma_priv dma_master;
+ struct nouveau_dma_priv dma_bufmgr;
+ struct nouveau_dma_priv *dma;
struct nouveau_fence *fence_head;
struct nouveau_fence *fence_tail;
uint32_t fence_sequence;
- struct nouveau_resource *pb_heap;
- struct nouveau_pushbuf *pb_head;
- struct nouveau_pushbuf *pb_tail;
+ struct nouveau_pushbuf_priv pb;
unsigned user_charge;
};
#include "nouveau_drmif.h"
#include "nouveau_dma.h"
-#define PB_RSVD_DWORDS 2
+#define PB_BUFMGR_DWORDS (4096 / 2)
+#define PB_MIN_USER_DWORDS 2048
static int
-nouveau_pushbuf_space(struct nouveau_channel *chan)
+nouveau_pushbuf_space(struct nouveau_channel *chan, unsigned min)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
- struct nouveau_pushbuf_priv *nvpb;
+ struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
- nvpb = calloc(1, sizeof(struct nouveau_pushbuf_priv));
- if (!nvpb)
- return -ENOMEM;
+ assert((min + 1) <= nvchan->dma->max);
- while (nouveau_resource_alloc(nvchan->pb_heap, 0x2100, NULL,
- &nvpb->res)) {
- nouveau_fence_flush(chan);
- }
+ /* Wait for enough space in push buffer */
+ min = min < PB_MIN_USER_DWORDS ? PB_MIN_USER_DWORDS : min;
+ min += 1; /* a bit extra for the NOP */
+ if (nvchan->dma->free < min)
+ WAIT_RING_CH(chan, min);
+
+ /* Insert NOP, may turn into a jump later */
+ RING_SPACE_CH(chan, 1);
+ nvpb->nop_jump = nvchan->dma->cur;
+ OUT_RING_CH(chan, 0);
+ /* Any remaining space is available to the user */
+ nvpb->start = nvchan->dma->cur;
+ nvpb->size = nvchan->dma->free;
nvpb->base.channel = chan;
- nvpb->base.remaining = (nvpb->res->size / 4) - PB_RSVD_DWORDS;
- nvpb->base.cur = &nvchan->pushbuf[nvpb->res->start/4];
- nvchan->pb_tail = &nvpb->base;
- nvchan->base.pushbuf = nvchan->pb_tail;
+ nvpb->base.remaining = nvpb->size;
+ nvpb->base.cur = &nvchan->pushbuf[nvpb->start];
return 0;
}
nouveau_pushbuf_init(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
+ struct nouveau_dma_priv *m = &nvchan->dma_master;
+ struct nouveau_dma_priv *b = &nvchan->dma_bufmgr;
+ int i;
if (!nvchan)
return -EINVAL;
- /* Everything except first 4KiB of the push buffer is managed by us */
- if (nouveau_resource_init(&nvchan->pb_heap, 4096,
- nvchan->drm.cmdbuf_size - 4096))
- return -EINVAL;
-
- /* Shrink master ring to 4KiB */
- assert(nvchan->dma.cur <= (4096/4));
- nvchan->dma.max = (4096 / 4) - 2;
- nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
-
- assert(!nouveau_pushbuf_space(chan));
+ /* Reassign last bit of push buffer for a "separate" bufmgr
+ * ring buffer
+ */
+ m->max -= PB_BUFMGR_DWORDS;
+ m->free -= PB_BUFMGR_DWORDS;
+
+ b->base = m->base + ((m->max + 2) << 2);
+ b->max = PB_BUFMGR_DWORDS - 2;
+ b->cur = b->put = 0;
+ b->free = b->max - b->cur;
+
+ /* Some NOPs just to be safe
+ *XXX: RING_SKIPS
+ */
+ nvchan->dma = b;
+ RING_SPACE_CH(chan, 8);
+ for (i = 0; i < 8; i++)
+ OUT_RING_CH(chan, 0);
+ nvchan->dma = m;
+
+ nouveau_pushbuf_space(chan, 0);
+ chan->pushbuf = &nvchan->pb.base;
return 0;
}
-static void
-nouveau_pushbuf_fence_signalled(void *priv)
-{
- struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(priv);
-
- nouveau_fence_del(&nvpb->fence);
- nouveau_resource_free(&nvpb->res);
- free(nvpb);
-}
-
/* This would be our TTM "superioctl" */
int
-nouveau_pushbuf_flush(struct nouveau_channel *chan)
+nouveau_pushbuf_flush(struct nouveau_channel *chan, unsigned min)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
- struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
+ struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
struct nouveau_pushbuf_bo *pbbo;
struct nouveau_fence *fence = NULL;
int ret;
- if (nvpb->base.remaining == (nvpb->res->size / 4) - PB_RSVD_DWORDS)
+ if (nvpb->base.remaining == nvpb->size)
return 0;
- nvchan->pb_tail = NULL;
+
+ nvpb->size -= nvpb->base.remaining;
+ nvchan->dma->cur += nvpb->size;
+ nvchan->dma->free -= nvpb->size;
+ assert(nvchan->dma->cur <= nvchan->dma->max);
ret = nouveau_fence_new(chan, &fence);
if (ret)
return ret;
+ nvchan->dma = &nvchan->dma_bufmgr;
+ nvchan->pushbuf[nvpb->nop_jump] = 0x20000000 |
+ (nvchan->dma->base + (nvchan->dma->cur << 2));
+
/* Validate buffers + apply relocations */
nvchan->user_charge = 0;
while ((pbbo = ptr_to_pbbo(nvpb->buffers))) {
}
nvpb->nr_buffers = 0;
- /* Emit JMP to indirect pushbuf */
+ /* Switch back to user's ring */
RING_SPACE_CH(chan, 1);
- OUT_RING_CH(chan, 0x20000000 | nvpb->res->start);
+ OUT_RING_CH(chan, 0x20000000 | ((nvpb->start << 2) +
+ nvchan->dma_master.base));
+ nvchan->dma = &nvchan->dma_master;
- /* Add JMP back to master pushbuf from indirect pushbuf */
- (*nvpb->base.cur++) =
- 0x20000000 | ((nvchan->dma.cur << 2) + nvchan->dma.base);
-
- /* Fence */
- nvpb->fence = fence;
- nouveau_fence_signal_cb(nvpb->fence, nouveau_pushbuf_fence_signalled,
- nvpb);
- nouveau_fence_emit(nvpb->fence);
-
- /* Kickoff */
+ /* Fence + kickoff */
+ nouveau_fence_emit(fence);
FIRE_RING_CH(chan);
+ nouveau_fence_del(&fence);
/* Allocate space for next push buffer */
- assert(!nouveau_pushbuf_space(chan));
+ assert(!nouveau_pushbuf_space(chan, min));
return 0;
}
static struct nouveau_pushbuf_bo *
nouveau_pushbuf_emit_buffer(struct nouveau_channel *chan, struct nouveau_bo *bo)
{
- struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
- struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(nvchan->pb_tail);
+ struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
struct nouveau_pushbuf_bo *pbbo = ptr_to_pbbo(nvpb->buffers);
while (pbbo) {
struct nouveau_channel *chan = grobj->channel;
uint32_t *pushbuf;
- if (chan->pushbuf->remaining < (size + 1))
- nouveau_pushbuf_flush(chan);
+ if (chan->pushbuf->remaining < (size + 1)) {
+ nouveau_pushbuf_flush(chan, size + 1);
+ }
pushbuf = chan->pushbuf->cur;
chan->pushbuf->cur += (size + 1);
void
nouveau_pipe_dma_kickoff(struct nouveau_channel *chan)
{
- nouveau_pushbuf_flush(chan);
+ nouveau_pushbuf_flush(chan, 0);
}
static int
projects / profile / ivi / mesa.git / commitdiff