* SPDX-License-Identifier: MIT
*/
+#include <xf86drm.h>
+#include "asahi/lib/decode.h"
#include "agx_state.h"
-#define foreach_batch(ctx, idx) \
+#define foreach_active(ctx, idx) \
BITSET_FOREACH_SET(idx, ctx->batches.active, AGX_MAX_BATCHES)
+#define foreach_submitted(ctx, idx) \
+ BITSET_FOREACH_SET(idx, ctx->batches.submitted, AGX_MAX_BATCHES)
+
static unsigned
agx_batch_idx(struct agx_batch *batch)
{
return BITSET_TEST(batch->ctx->batches.active, agx_batch_idx(batch));
}
+bool
+agx_batch_is_submitted(struct agx_batch *batch)
+{
+ return BITSET_TEST(batch->ctx->batches.submitted, agx_batch_idx(batch));
+}
+
+static void
+agx_batch_mark_active(struct agx_batch *batch)
+{
+ unsigned batch_idx = agx_batch_idx(batch);
+
+ assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
+ assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx));
+ BITSET_SET(batch->ctx->batches.active, batch_idx);
+}
+
+static void
+agx_batch_mark_submitted(struct agx_batch *batch)
+{
+ unsigned batch_idx = agx_batch_idx(batch);
+
+ assert(BITSET_TEST(batch->ctx->batches.active, batch_idx));
+ assert(!BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
+ BITSET_CLEAR(batch->ctx->batches.active, batch_idx);
+ BITSET_SET(batch->ctx->batches.submitted, batch_idx);
+}
+
+static void
+agx_batch_mark_complete(struct agx_batch *batch)
+{
+ unsigned batch_idx = agx_batch_idx(batch);
+
+ assert(!BITSET_TEST(batch->ctx->batches.active, batch_idx));
+ assert(BITSET_TEST(batch->ctx->batches.submitted, batch_idx));
+ BITSET_CLEAR(batch->ctx->batches.submitted, batch_idx);
+}
+
static void
agx_batch_init(struct agx_context *ctx,
const struct pipe_framebuffer_state *key,
agx_batch_writes(batch, agx_resource(key->cbufs[i]->texture));
}
- unsigned batch_idx = agx_batch_idx(batch);
- BITSET_SET(ctx->batches.active, batch_idx);
-
if (key->width != AGX_COMPUTE_BATCH_WIDTH)
agx_batch_init_state(batch);
+
+ if (!batch->syncobj) {
+ int ret = drmSyncobjCreate(dev->fd, 0, &batch->syncobj);
+ assert(!ret && batch->syncobj);
+ }
+
+ agx_batch_mark_active(batch);
+}
+
+static void
+agx_batch_print_stats(struct agx_device *dev, struct agx_batch *batch)
+{
+ unreachable("Linux UAPI not yet upstream");
}
void
{
struct agx_device *dev = agx_device(ctx->base.screen);
assert(batch->ctx == ctx);
+ assert(agx_batch_is_submitted(batch));
- if (ctx->batch == batch)
- ctx->batch = NULL;
+ assert(ctx->batch != batch);
agx_finish_batch_occlusion_queries(batch);
batch->occlusion_buffer.cpu = NULL;
int handle;
AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
+ struct agx_bo *bo = agx_lookup_bo(dev, handle);
+
/* There is no more writer on this context for anything we wrote */
struct agx_batch *writer = agx_writer_get(ctx, handle);
- assert((writer == NULL || writer == batch) &&
- "cannot read while another batch writes");
- agx_writer_remove(ctx, handle);
+ if (writer == batch) {
+ bo->writer_syncobj = 0;
+ agx_writer_remove(ctx, handle);
+ }
+
agx_bo_unreference(agx_lookup_bo(dev, handle));
}
util_dynarray_fini(&batch->occlusion_queries);
util_unreference_framebuffer_state(&batch->key);
- unsigned batch_idx = agx_batch_idx(batch);
- BITSET_CLEAR(ctx->batches.active, batch_idx);
+ agx_batch_mark_complete(batch);
+
+ if (!(dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC))) {
+ agx_batch_print_stats(dev, batch);
+ }
+}
+
+int
+agx_cleanup_batches(struct agx_context *ctx)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+
+ unsigned i;
+ unsigned count = 0;
+ struct agx_batch *batches[AGX_MAX_BATCHES];
+ uint32_t syncobjs[AGX_MAX_BATCHES];
+ uint32_t first = 0;
+
+ foreach_submitted(ctx, i) {
+ batches[count] = &ctx->batches.slots[i];
+ syncobjs[count++] = ctx->batches.slots[i].syncobj;
+ }
+
+ if (!count)
+ return -1;
+
+ int ret = drmSyncobjWait(dev->fd, syncobjs, count, 0, 0, &first);
+ assert(!ret || ret == -ETIME);
+ if (ret)
+ return -1;
+
+ assert(first < AGX_MAX_BATCHES);
+ agx_batch_cleanup(ctx, batches[first]);
+ return agx_batch_idx(batches[first]);
}
static struct agx_batch *
{
/* Look if we have a matching batch */
unsigned i;
- foreach_batch(ctx, i) {
+ foreach_active(ctx, i) {
struct agx_batch *candidate = &ctx->batches.slots[i];
if (util_framebuffer_state_equal(&candidate->key, state)) {
}
}
- /* Look if we have a free batch */
- struct agx_batch *batch = NULL;
- for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
- if (!BITSET_TEST(ctx->batches.active, i)) {
- batch = &ctx->batches.slots[i];
- break;
+ /* Look for a free batch */
+ for (i = 0; i < AGX_MAX_BATCHES; ++i) {
+ if (!BITSET_TEST(ctx->batches.active, i) &&
+ !BITSET_TEST(ctx->batches.submitted, i)) {
+ struct agx_batch *batch = &ctx->batches.slots[i];
+ agx_batch_init(ctx, state, batch);
+ return batch;
}
}
+ /* Try to clean up one batch */
+ int freed = agx_cleanup_batches(ctx);
+ if (freed >= 0) {
+ struct agx_batch *batch = &ctx->batches.slots[freed];
+ agx_batch_init(ctx, state, batch);
+ return batch;
+ }
+
/* Else, evict something */
- if (!batch) {
- for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
- struct agx_batch *candidate = &ctx->batches.slots[i];
+ struct agx_batch *batch = NULL;
+ bool submitted = false;
+ for (i = 0; i < AGX_MAX_BATCHES; ++i) {
+ struct agx_batch *candidate = &ctx->batches.slots[i];
+ bool cand_submitted = BITSET_TEST(ctx->batches.submitted, i);
- if (!batch || batch->seqnum > candidate->seqnum)
- batch = candidate;
- }
+ /* Prefer submitted batches first */
+ if (!cand_submitted && submitted)
+ continue;
- agx_flush_batch(ctx, batch);
+ if (!batch || batch->seqnum > candidate->seqnum) {
+ batch = candidate;
+ submitted = cand_submitted;
+ }
}
+ assert(batch);
+
+ agx_sync_batch_for_reason(ctx, batch, "Too many batches");
/* Batch is now free */
agx_batch_init(ctx, state, batch);
void
agx_flush_all(struct agx_context *ctx, const char *reason)
{
- if (reason)
- perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
-
unsigned idx;
- foreach_batch(ctx, idx) {
+ foreach_active(ctx, idx) {
+ if (reason)
+ perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
+
agx_flush_batch(ctx, &ctx->batches.slots[idx]);
}
}
if (reason)
perf_debug_ctx(ctx, "Flushing due to: %s\n", reason);
- agx_flush_batch(ctx, batch);
+ if (agx_batch_is_active(batch))
+ agx_flush_batch(ctx, batch);
}
static void
agx_flush_readers_except(struct agx_context *ctx, struct agx_resource *rsrc,
- struct agx_batch *except, const char *reason)
+ struct agx_batch *except, const char *reason,
+ bool sync)
{
unsigned idx;
- foreach_batch(ctx, idx) {
+ /* Flush everything to the hardware first */
+ foreach_active(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (batch == except)
agx_flush_batch(ctx, batch);
}
}
+
+ /* Then wait on everything if necessary */
+ if (sync) {
+ foreach_submitted(ctx, idx) {
+ struct agx_batch *batch = &ctx->batches.slots[idx];
+
+ if (batch == except)
+ continue;
+
+ if (agx_batch_uses_bo(batch, rsrc->bo)) {
+ perf_debug_ctx(ctx, "Sync reader due to: %s\n", reason);
+ agx_sync_batch(ctx, batch);
+ }
+ }
+ }
}
static void
agx_flush_writer_except(struct agx_context *ctx, struct agx_resource *rsrc,
- struct agx_batch *except, const char *reason)
+ struct agx_batch *except, const char *reason, bool sync)
{
struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle);
- if (writer && writer != except) {
- perf_debug_ctx(ctx, "Flush writer due to: %s\n", reason);
- agx_flush_batch(ctx, writer);
+ if (writer && writer != except &&
+ (agx_batch_is_active(writer) || agx_batch_is_submitted(writer))) {
+ if (agx_batch_is_active(writer) || sync) {
+ perf_debug_ctx(ctx, "%s writer due to: %s\n", sync ? "Sync" : "Flush",
+ reason);
+ }
+ if (agx_batch_is_active(writer))
+ agx_flush_batch(ctx, writer);
+ /* Check for submitted state, because if the batch was a no-op it'll
+ * already be cleaned up */
+ if (sync && agx_batch_is_submitted(writer))
+ agx_sync_batch(ctx, writer);
}
}
agx_any_batch_uses_resource(struct agx_context *ctx, struct agx_resource *rsrc)
{
unsigned idx;
- foreach_batch(ctx, idx) {
+ foreach_active(ctx, idx) {
+ struct agx_batch *batch = &ctx->batches.slots[idx];
+
+ if (agx_batch_uses_bo(batch, rsrc->bo))
+ return true;
+ }
+
+ foreach_submitted(ctx, idx) {
struct agx_batch *batch = &ctx->batches.slots[idx];
if (agx_batch_uses_bo(batch, rsrc->bo))
agx_flush_readers(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
- agx_flush_readers_except(ctx, rsrc, NULL, reason);
+ agx_flush_readers_except(ctx, rsrc, NULL, reason, false);
+}
+
+void
+agx_sync_readers(struct agx_context *ctx, struct agx_resource *rsrc,
+ const char *reason)
+{
+ agx_flush_readers_except(ctx, rsrc, NULL, reason, true);
}
void
agx_flush_writer(struct agx_context *ctx, struct agx_resource *rsrc,
const char *reason)
{
- agx_flush_writer_except(ctx, rsrc, NULL, reason);
+ agx_flush_writer_except(ctx, rsrc, NULL, reason, false);
+}
+
+void
+agx_sync_writer(struct agx_context *ctx, struct agx_resource *rsrc,
+ const char *reason)
+{
+ agx_flush_writer_except(ctx, rsrc, NULL, reason, true);
}
void
agx_batch_reads(struct agx_batch *batch, struct agx_resource *rsrc)
{
/* Hazard: read-after-write */
- agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch");
+ agx_flush_writer_except(batch->ctx, rsrc, batch, "Read from another batch",
+ false);
agx_batch_add_bo(batch, rsrc->bo);
struct agx_context *ctx = batch->ctx;
struct agx_batch *writer = agx_writer_get(ctx, rsrc->bo->handle);
- agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch");
+ agx_flush_readers_except(ctx, rsrc, batch, "Write from other batch", false);
/* Nothing to do if we're already writing */
if (writer == batch)
/* Write is strictly stronger than a read */
agx_batch_reads(batch, rsrc);
- /* We are now the new writer */
+ writer = agx_writer_get(ctx, rsrc->bo->handle);
+ assert(!writer || agx_batch_is_submitted(writer));
+
+ /* We are now the new writer. Disregard the previous writer -- anything that
+ * needs to wait for the writer going forward needs to wait for us.
+ */
+ agx_writer_remove(ctx, rsrc->bo->handle);
agx_writer_add(ctx, agx_batch_idx(batch), rsrc->bo->handle);
}
agx_flush_occlusion_queries(struct agx_context *ctx)
{
unsigned i;
- foreach_batch(ctx, i) {
+ foreach_active(ctx, i) {
struct agx_batch *other = &ctx->batches.slots[i];
if (other->occlusion_queries.size != 0)
agx_flush_batch_for_reason(ctx, other, "Occlusion query ordering");
}
+
+ foreach_submitted(ctx, i) {
+ struct agx_batch *other = &ctx->batches.slots[i];
+
+ if (other->occlusion_queries.size != 0)
+ agx_sync_batch_for_reason(ctx, other, "Occlusion query ordering");
+ }
+}
+
+static int
+agx_get_in_sync(struct agx_context *ctx)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+
+ if (ctx->in_sync_fd >= 0) {
+ int ret =
+ drmSyncobjImportSyncFile(dev->fd, ctx->in_sync_obj, ctx->in_sync_fd);
+ assert(!ret);
+
+ close(ctx->in_sync_fd);
+ ctx->in_sync_fd = -1;
+
+ return ctx->in_sync_obj;
+ } else {
+ return 0;
+ }
+}
+
+static void
+agx_add_sync(struct drm_asahi_sync *syncs, unsigned *count, uint32_t handle)
+{
+ if (!handle)
+ return;
+
+ syncs[(*count)++] = (struct drm_asahi_sync){
+ .sync_type = DRM_ASAHI_SYNC_SYNCOBJ,
+ .handle = handle,
+ };
+}
+
+void
+agx_batch_submit(struct agx_context *ctx, struct agx_batch *batch,
+ uint32_t barriers, enum drm_asahi_cmd_type cmd_type,
+ void *cmdbuf)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+
+ bool feedback = dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC | AGX_DBG_STATS);
+
+#ifndef NDEBUG
+ /* Debug builds always get feedback (for fault checks) */
+ feedback = true;
+#endif
+
+ if (!feedback)
+ batch->result = NULL;
+
+ /* We allocate the worst-case sync array size since this won't be excessive
+ * for most workloads
+ */
+ unsigned max_syncs = agx_batch_bo_list_bits(batch) + 1;
+ unsigned in_sync_count = 0;
+ unsigned shared_bo_count = 0;
+ struct drm_asahi_sync *in_syncs =
+ malloc(max_syncs * sizeof(struct drm_asahi_sync));
+ struct agx_bo **shared_bos = malloc(max_syncs * sizeof(struct agx_bo *));
+
+ struct drm_asahi_sync out_sync = {
+ .sync_type = DRM_ASAHI_SYNC_SYNCOBJ,
+ .handle = batch->syncobj,
+ };
+
+ int handle;
+ AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
+ struct agx_bo *bo = agx_lookup_bo(dev, handle);
+
+ if (bo->flags & AGX_BO_SHARED) {
+ /* Get a sync file fd from the buffer */
+ int in_sync_fd = agx_export_sync_file(dev, bo);
+ assert(in_sync_fd >= 0);
+
+ /* Create a new syncobj */
+ uint32_t sync_handle;
+ int ret = drmSyncobjCreate(dev->fd, 0, &sync_handle);
+ assert(ret >= 0);
+
+ /* Import the sync file into it */
+ ret = drmSyncobjImportSyncFile(dev->fd, sync_handle, in_sync_fd);
+ assert(ret >= 0);
+ assert(sync_handle);
+ close(in_sync_fd);
+
+ /* Add it to our wait list */
+ agx_add_sync(in_syncs, &in_sync_count, sync_handle);
+
+ /* And keep track of the BO for cloning the out_sync */
+ shared_bos[shared_bo_count++] = bo;
+ }
+ }
+
+ /* Add an explicit fence from gallium, if any */
+ agx_add_sync(in_syncs, &in_sync_count, agx_get_in_sync(ctx));
+
+ /* Submit! */
+ agx_submit_single(
+ dev, cmd_type, barriers, in_syncs, in_sync_count, &out_sync, 1, cmdbuf,
+ feedback ? ctx->result_buf->handle : 0, feedback ? batch->result_off : 0,
+ feedback ? sizeof(union agx_batch_result) : 0);
+
+ /* Now stash our batch fence into any shared BOs. */
+ if (shared_bo_count) {
+ /* Convert our handle to a sync file */
+ int out_sync_fd = -1;
+ int ret = drmSyncobjExportSyncFile(dev->fd, batch->syncobj, &out_sync_fd);
+ assert(ret >= 0);
+ assert(out_sync_fd >= 0);
+
+ for (unsigned i = 0; i < shared_bo_count; i++) {
+ /* Free the in_sync handle we just acquired */
+ ret = drmSyncobjDestroy(dev->fd, in_syncs[i].handle);
+ assert(ret >= 0);
+ /* And then import the out_sync sync file into it */
+ ret = agx_import_sync_file(dev, shared_bos[i], out_sync_fd);
+ assert(ret >= 0);
+ }
+
+ close(out_sync_fd);
+ }
+
+ /* Record the syncobj on each BO we write, so it can be added post-facto as a
+ * fence if the BO is exported later...
+ */
+ AGX_BATCH_FOREACH_BO_HANDLE(batch, handle) {
+ struct agx_bo *bo = agx_lookup_bo(dev, handle);
+ struct agx_batch *writer = agx_writer_get(ctx, handle);
+
+ if (!writer)
+ continue;
+
+ /* Skip BOs that are written by submitted batches, they're not ours */
+ if (agx_batch_is_submitted(writer))
+ continue;
+
+ /* But any BOs written by active batches are ours */
+ assert(writer == batch && "exclusive writer");
+ bo->writer_syncobj = batch->syncobj;
+ }
+
+ free(in_syncs);
+ free(shared_bos);
+
+ if (dev->debug & (AGX_DBG_TRACE | AGX_DBG_SYNC)) {
+ /* Wait so we can get errors reported back */
+ int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL);
+ assert(!ret);
+
+ if (dev->debug & AGX_DBG_TRACE) {
+ /* agxdecode DRM commands */
+ switch (cmd_type) {
+ default:
+ unreachable("Linux UAPI not yet upstream");
+ }
+ agxdecode_next_frame();
+ }
+
+ agx_batch_print_stats(dev, batch);
+ }
+
+ agx_batch_mark_submitted(batch);
+
+ /* Record the last syncobj for fence creation */
+ ctx->syncobj = batch->syncobj;
+
+ if (ctx->batch == batch)
+ ctx->batch = NULL;
+
+ /* Try to clean up up to two batches, to keep memory usage down */
+ if (agx_cleanup_batches(ctx) >= 0)
+ agx_cleanup_batches(ctx);
+}
+
+void
+agx_sync_batch(struct agx_context *ctx, struct agx_batch *batch)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+
+ if (agx_batch_is_active(batch))
+ agx_flush_batch(ctx, batch);
+
+ /* Empty batch case, already cleaned up */
+ if (!agx_batch_is_submitted(batch))
+ return;
+
+ assert(batch->syncobj);
+ int ret = drmSyncobjWait(dev->fd, &batch->syncobj, 1, INT64_MAX, 0, NULL);
+ assert(!ret);
+ agx_batch_cleanup(ctx, batch);
+}
+
+void
+agx_sync_batch_for_reason(struct agx_context *ctx, struct agx_batch *batch,
+ const char *reason)
+{
+ if (reason)
+ perf_debug_ctx(ctx, "Syncing due to: %s\n", reason);
+
+ agx_sync_batch(ctx, batch);
+}
+
+void
+agx_sync_all(struct agx_context *ctx, const char *reason)
+{
+ if (reason)
+ perf_debug_ctx(ctx, "Syncing all due to: %s\n", reason);
+
+ unsigned idx;
+ foreach_active(ctx, idx) {
+ agx_flush_batch(ctx, &ctx->batches.slots[idx]);
+ }
+
+ foreach_submitted(ctx, idx) {
+ agx_sync_batch(ctx, &ctx->batches.slots[idx]);
+ }
+}
+
+void
+agx_batch_reset(struct agx_context *ctx, struct agx_batch *batch)
+{
+ /* Reset an empty batch. Like submit, but does nothing. */
+ agx_batch_mark_submitted(batch);
+
+ if (ctx->batch == batch)
+ ctx->batch = NULL;
+
+ agx_batch_cleanup(ctx, batch);
}
--- /dev/null
+/*
+ * Based on panfrost/pan_fence.c:
+ *
+ * Copyright (c) 2022 Amazon.com, Inc. or its affiliates.
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018 Alyssa Rosenzweig
+ * Copyright (C) 2019 Collabora, Ltd.
+ * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <xf86drm.h>
+
+#include "agx_fence.h"
+#include "agx_state.h"
+
+#include "util/os_time.h"
+#include "util/u_inlines.h"
+
+void
+agx_fence_reference(struct pipe_screen *pscreen, struct pipe_fence_handle **ptr,
+ struct pipe_fence_handle *fence)
+{
+ struct agx_device *dev = agx_device(pscreen);
+ struct pipe_fence_handle *old = *ptr;
+
+ if (pipe_reference(old ? &old->reference : NULL,
+ fence ? &fence->reference : NULL)) {
+ drmSyncobjDestroy(dev->fd, old->syncobj);
+ free(old);
+ }
+
+ *ptr = fence;
+}
+
+bool
+agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx,
+ struct pipe_fence_handle *fence, uint64_t timeout)
+{
+ struct agx_device *dev = agx_device(pscreen);
+ int ret;
+
+ if (fence->signaled)
+ return true;
+
+ uint64_t abs_timeout = os_time_get_absolute_timeout(timeout);
+ if (abs_timeout == OS_TIMEOUT_INFINITE)
+ abs_timeout = INT64_MAX;
+
+ ret = drmSyncobjWait(dev->fd, &fence->syncobj, 1, abs_timeout,
+ DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL, NULL);
+
+ assert(ret >= 0 || ret == -ETIME);
+ fence->signaled = (ret >= 0);
+ return fence->signaled;
+}
+
+int
+agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f)
+{
+ struct agx_device *dev = agx_device(screen);
+ int fd = -1;
+
+ int ret = drmSyncobjExportSyncFile(dev->fd, f->syncobj, &fd);
+ assert(ret >= 0);
+ assert(fd >= 0);
+
+ return fd;
+}
+
+struct pipe_fence_handle *
+agx_fence_from_fd(struct agx_context *ctx, int fd, enum pipe_fd_type type)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+ int ret;
+
+ struct pipe_fence_handle *f = calloc(1, sizeof(*f));
+ if (!f)
+ return NULL;
+
+ if (type == PIPE_FD_TYPE_NATIVE_SYNC) {
+ ret = drmSyncobjCreate(dev->fd, 0, &f->syncobj);
+ if (ret) {
+ fprintf(stderr, "create syncobj failed\n");
+ goto err_free_fence;
+ }
+
+ ret = drmSyncobjImportSyncFile(dev->fd, f->syncobj, fd);
+ if (ret) {
+ fprintf(stderr, "import syncfile failed\n");
+ goto err_destroy_syncobj;
+ }
+ } else {
+ assert(type == PIPE_FD_TYPE_SYNCOBJ);
+ ret = drmSyncobjFDToHandle(dev->fd, fd, &f->syncobj);
+ if (ret) {
+ fprintf(stderr, "import syncobj FD failed\n");
+ goto err_free_fence;
+ }
+ }
+
+ pipe_reference_init(&f->reference, 1);
+
+ return f;
+
+err_destroy_syncobj:
+ drmSyncobjDestroy(dev->fd, f->syncobj);
+err_free_fence:
+ free(f);
+ return NULL;
+}
+
+struct pipe_fence_handle *
+agx_fence_create(struct agx_context *ctx)
+{
+ struct agx_device *dev = agx_device(ctx->base.screen);
+ int fd = -1, ret;
+
+ /* Snapshot the last rendering out fence. We'd rather have another
+ * syncobj instead of a sync file, but this is all we get.
+ * (HandleToFD/FDToHandle just gives you another syncobj ID for the
+ * same syncobj).
+ */
+ ret = drmSyncobjExportSyncFile(dev->fd, ctx->syncobj, &fd);
+ assert(ret >= 0 && fd != -1 && "export failed");
+ if (ret || fd == -1) {
+ fprintf(stderr, "export failed\n");
+ return NULL;
+ }
+
+ struct pipe_fence_handle *f =
+ agx_fence_from_fd(ctx, fd, PIPE_FD_TYPE_NATIVE_SYNC);
+
+ close(fd);
+
+ return f;
+}
*/
#include <errno.h>
#include <stdio.h>
+#include <xf86drm.h>
#include "asahi/compiler/agx_compile.h"
#include "asahi/layout/layout.h"
#include "asahi/lib/agx_formats.h"
#include "util/u_upload_mgr.h"
#include "agx_device.h"
#include "agx_disk_cache.h"
+#include "agx_fence.h"
#include "agx_public.h"
#include "agx_state.h"
/*
* Perform the required synchronization before a transfer_map operation can
- * complete. This may require flushing batches.
+ * complete. This may require syncing batches.
*/
static void
agx_prepare_for_map(struct agx_context *ctx, struct agx_resource *rsrc,
if (usage & PIPE_MAP_UNSYNCHRONIZED)
return;
- /* Both writing and reading need writers flushed */
- agx_flush_writer(ctx, rsrc, "Unsynchronized transfer");
+ /* Both writing and reading need writers synced */
+ agx_sync_writer(ctx, rsrc, "Unsynchronized transfer");
- /* Additionally, writing needs readers flushed */
+ /* Additionally, writing needs readers synced */
if (!(usage & PIPE_MAP_WRITE))
return;
if (!agx_any_batch_uses_resource(ctx, rsrc))
return;
- /* There are readers. Try to shadow the resource to avoid a flush */
+ /* There are readers. Try to shadow the resource to avoid a sync */
if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) && agx_shadow(ctx, rsrc))
return;
- /* Otherwise, we need to flush */
- agx_flush_readers(ctx, rsrc, "Unsynchronized write");
+ /* Otherwise, we need to sync */
+ agx_sync_readers(ctx, rsrc, "Unsynchronized write");
}
/* Most of the time we can do CPU-side transfers, but sometimes we need to use
if ((usage & PIPE_MAP_READ) && agx_resource_valid(rsrc, level)) {
agx_blit_to_staging(pctx, transfer);
- agx_flush_writer(ctx, staging, "GPU read staging blit");
+ agx_sync_writer(ctx, staging, "GPU read staging blit");
}
agx_bo_mmap(staging->bo);
pres->bind |= PIPE_BIND_SHARED;
agx_flush_writer(agx_context(pctx), rsrc, "flush_resource");
}
-
}
/*
{
struct agx_context *ctx = agx_context(pctx);
- if (fence)
- *fence = NULL;
-
agx_flush_all(ctx, "Gallium flush");
+
+ /* At this point all pending work has been submitted. Since jobs are
+ * started and completed sequentially from a UAPI perspective, and since
+ * we submit all jobs with compute+render barriers on the prior job,
+ * waiting on the last submitted job is sufficient to guarantee completion
+ * of all GPU work thus far, so we can create a fence out of the latest
+ * syncobj.
+ *
+ * See this page for more info on how the GPU/UAPI queueing works:
+ * https://github.com/AsahiLinux/docs/wiki/SW:AGX-driver-notes#queues
+ */
+
+ if (fence) {
+ struct pipe_fence_handle *f = agx_fence_create(ctx);
+ pctx->screen->fence_reference(pctx->screen, fence, NULL);
+ *fence = f;
+ }
}
void
struct agx_device *dev = agx_device(ctx->base.screen);
assert(agx_batch_is_active(batch));
+ assert(!agx_batch_is_submitted(batch));
/* Make sure there's something to submit. */
if (!batch->clear && !batch->any_draws) {
- agx_batch_cleanup(ctx, batch);
+ agx_batch_reset(ctx, batch);
return;
}
(void)pipeline_background;
(void)pipeline_background_partial;
- agx_batch_cleanup(ctx, batch);
+ unreachable("Linux UAPI not yet upstream");
+ agx_batch_submit(ctx, batch, 0, 0, NULL);
}
static void
agx_destroy_context(struct pipe_context *pctx)
{
+ struct agx_device *dev = agx_device(pctx->screen);
struct agx_context *ctx = agx_context(pctx);
+ /* Batch state needs to be freed on completion, and we don't want to yank
+ * buffers out from in-progress GPU jobs to avoid faults, so just wait until
+ * everything in progress is actually done on context destroy. This will
+ * ensure everything is cleaned up properly.
+ */
+ agx_sync_all(ctx, "destroy context");
+
if (pctx->stream_uploader)
u_upload_destroy(pctx->stream_uploader);
agx_bo_unreference(ctx->result_buf);
+ drmSyncobjDestroy(dev->fd, ctx->in_sync_obj);
+ drmSyncobjDestroy(dev->fd, ctx->dummy_syncobj);
+ if (ctx->in_sync_fd != -1)
+ close(ctx->in_sync_fd);
+
+ for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
+ if (ctx->batches.slots[i].syncobj)
+ drmSyncobjDestroy(dev->fd, ctx->batches.slots[i].syncobj);
+ }
+
ralloc_free(ctx);
}
{
struct agx_context *ctx = rzalloc(NULL, struct agx_context);
struct pipe_context *pctx = &ctx->base;
+ int ret;
if (!ctx)
return NULL;
ctx->blitter = util_blitter_create(pctx);
ctx->result_buf = agx_bo_create(
- agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES, 0,
- "Batch result buffer");
+ agx_device(screen), sizeof(union agx_batch_result) * AGX_MAX_BATCHES,
+ AGX_BO_WRITEBACK, "Batch result buffer");
assert(ctx->result_buf);
+ /* Sync object/FD used for NATIVE_FENCE_FD. */
+ ctx->in_sync_fd = -1;
+ ret = drmSyncobjCreate(agx_device(screen)->fd, 0, &ctx->in_sync_obj);
+ assert(!ret);
+
+ /* Dummy sync object used before any work has been submitted. */
+ ret = drmSyncobjCreate(agx_device(screen)->fd, DRM_SYNCOBJ_CREATE_SIGNALED,
+ &ctx->dummy_syncobj);
+ assert(!ret);
+ ctx->syncobj = ctx->dummy_syncobj;
+
return pctx;
}
case PIPE_CAP_PRIMITIVE_RESTART:
case PIPE_CAP_PRIMITIVE_RESTART_FIXED_INDEX:
case PIPE_CAP_ANISOTROPIC_FILTER:
+ case PIPE_CAP_NATIVE_FENCE_FD:
return true;
case PIPE_CAP_SAMPLER_VIEW_TARGET:
ralloc_free(screen);
}
-static void
-agx_fence_reference(struct pipe_screen *screen, struct pipe_fence_handle **ptr,
- struct pipe_fence_handle *fence)
-{
-}
-
-static bool
-agx_fence_finish(struct pipe_screen *screen, struct pipe_context *ctx,
- struct pipe_fence_handle *fence, uint64_t timeout)
-{
- return true;
-}
-
static const void *
agx_get_compiler_options(struct pipe_screen *pscreen, enum pipe_shader_ir ir,
enum pipe_shader_type shader)
screen->get_timestamp = u_default_get_timestamp;
screen->fence_reference = agx_fence_reference;
screen->fence_finish = agx_fence_finish;
+ screen->fence_get_fd = agx_fence_get_fd;
screen->get_compiler_options = agx_get_compiler_options;
screen->get_disk_shader_cache = agx_get_disk_shader_cache;
projects / platform / upstream / mesa.git / commitdiff