/* * Copyright © 2018 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ /** * @file iris_fence.c * * Fences for driver and IPC serialisation, scheduling and synchronisation. */ #include "drm-uapi/sync_file.h" #include "util/u_debug.h" #include "util/u_inlines.h" #include "intel/common/intel_gem.h" #include "iris_batch.h" #include "iris_bufmgr.h" #include "iris_context.h" #include "iris_fence.h" #include "iris_screen.h" static uint32_t gem_syncobj_create(int fd, uint32_t flags) { struct drm_syncobj_create args = { .flags = flags, }; intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args); return args.handle; } static void gem_syncobj_destroy(int fd, uint32_t handle) { struct drm_syncobj_destroy args = { .handle = handle, }; intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args); } /** * Make a new sync-point. */ struct iris_syncobj * iris_create_syncobj(struct iris_bufmgr *bufmgr) { int fd = iris_bufmgr_get_fd(bufmgr); struct iris_syncobj *syncobj = malloc(sizeof(*syncobj)); if (!syncobj) return NULL; syncobj->handle = gem_syncobj_create(fd, 0); assert(syncobj->handle); pipe_reference_init(&syncobj->ref, 1); return syncobj; } void iris_syncobj_destroy(struct iris_bufmgr *bufmgr, struct iris_syncobj *syncobj) { int fd = iris_bufmgr_get_fd(bufmgr); gem_syncobj_destroy(fd, syncobj->handle); free(syncobj); } void iris_syncobj_signal(struct iris_bufmgr *bufmgr, struct iris_syncobj *syncobj) { int fd = iris_bufmgr_get_fd(bufmgr); struct drm_syncobj_array args = { .handles = (uintptr_t)&syncobj->handle, .count_handles = 1, }; if (intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_SIGNAL, &args)) { fprintf(stderr, "failed to signal syncobj %"PRIu32"\n", syncobj->handle); } } /** * Add a sync-point to the batch, with the given flags. * * \p flags One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL. */ void iris_batch_add_syncobj(struct iris_batch *batch, struct iris_syncobj *syncobj, unsigned flags) { struct drm_i915_gem_exec_fence *fence = util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1); *fence = (struct drm_i915_gem_exec_fence) { .handle = syncobj->handle, .flags = flags, }; struct iris_syncobj **store = util_dynarray_grow(&batch->syncobjs, struct iris_syncobj *, 1); *store = NULL; iris_syncobj_reference(batch->screen->bufmgr, store, syncobj); } /** * Walk through a batch's dependencies (any I915_EXEC_FENCE_WAIT syncobjs) * and unreference any which have already passed. * * Sometimes the compute batch is seldom used, and accumulates references * to stale render batches that are no longer of interest, so we can free * those up. */ static void clear_stale_syncobjs(struct iris_batch *batch) { struct iris_screen *screen = batch->screen; struct iris_bufmgr *bufmgr = screen->bufmgr; int n = util_dynarray_num_elements(&batch->syncobjs, struct iris_syncobj *); assert(n == util_dynarray_num_elements(&batch->exec_fences, struct drm_i915_gem_exec_fence)); /* Skip the first syncobj, as it's the signalling one. */ for (int i = n - 1; i > 1; i--) { struct iris_syncobj **syncobj = util_dynarray_element(&batch->syncobjs, struct iris_syncobj *, i); struct drm_i915_gem_exec_fence *fence = util_dynarray_element(&batch->exec_fences, struct drm_i915_gem_exec_fence, i); assert(fence->flags & I915_EXEC_FENCE_WAIT); if (iris_wait_syncobj(bufmgr, *syncobj, 0)) continue; /* This sync object has already passed, there's no need to continue * marking it as a dependency; we can stop holding on to the reference. */ iris_syncobj_reference(bufmgr, syncobj, NULL); /* Remove it from the lists; move the last element here. */ struct iris_syncobj **nth_syncobj = util_dynarray_pop_ptr(&batch->syncobjs, struct iris_syncobj *); struct drm_i915_gem_exec_fence *nth_fence = util_dynarray_pop_ptr(&batch->exec_fences, struct drm_i915_gem_exec_fence); if (syncobj != nth_syncobj) { *syncobj = *nth_syncobj; memcpy(fence, nth_fence, sizeof(*fence)); } } } /* ------------------------------------------------------------------- */ struct pipe_fence_handle { struct pipe_reference ref; struct pipe_context *unflushed_ctx; struct iris_fine_fence *fine[IRIS_BATCH_COUNT]; }; static void iris_fence_destroy(struct pipe_screen *p_screen, struct pipe_fence_handle *fence) { struct iris_screen *screen = (struct iris_screen *)p_screen; for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) iris_fine_fence_reference(screen, &fence->fine[i], NULL); free(fence); } static void iris_fence_reference(struct pipe_screen *p_screen, struct pipe_fence_handle **dst, struct pipe_fence_handle *src) { if (pipe_reference(*dst ? &(*dst)->ref : NULL, src ? &src->ref : NULL)) iris_fence_destroy(p_screen, *dst); *dst = src; } bool iris_wait_syncobj(struct iris_bufmgr *bufmgr, struct iris_syncobj *syncobj, int64_t timeout_nsec) { if (!syncobj) return false; int fd = iris_bufmgr_get_fd(bufmgr); struct drm_syncobj_wait args = { .handles = (uintptr_t)&syncobj->handle, .count_handles = 1, .timeout_nsec = timeout_nsec, }; return intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_WAIT, &args); } #define CSI "\e[" #define BLUE_HEADER CSI "0;97;44m" #define NORMAL CSI "0m" static void iris_fence_flush(struct pipe_context *ctx, struct pipe_fence_handle **out_fence, unsigned flags) { struct iris_screen *screen = (void *) ctx->screen; struct iris_context *ice = (struct iris_context *)ctx; /* We require DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT (kernel 5.2+) for * deferred flushes. Just ignore the request to defer on older kernels. */ if (!(screen->kernel_features & KERNEL_HAS_WAIT_FOR_SUBMIT)) flags &= ~PIPE_FLUSH_DEFERRED; const bool deferred = flags & PIPE_FLUSH_DEFERRED; if (flags & PIPE_FLUSH_END_OF_FRAME) { ice->frame++; if (INTEL_DEBUG(DEBUG_SUBMIT)) { fprintf(stderr, "%s ::: FRAME %-10u (ctx %p)%-35c%s\n", INTEL_DEBUG(DEBUG_COLOR) ? BLUE_HEADER : "", ice->frame, ctx, ' ', INTEL_DEBUG(DEBUG_COLOR) ? NORMAL : ""); } } iris_flush_dirty_dmabufs(ice); if (!deferred) { for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++) iris_batch_flush(&ice->batches[i]); } if (flags & PIPE_FLUSH_END_OF_FRAME) { iris_measure_frame_end(ice); } if (!out_fence) return; struct pipe_fence_handle *fence = calloc(1, sizeof(*fence)); if (!fence) return; pipe_reference_init(&fence->ref, 1); if (deferred) fence->unflushed_ctx = ctx; for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) { struct iris_batch *batch = &ice->batches[b]; if (deferred && iris_batch_bytes_used(batch) > 0) { struct iris_fine_fence *fine = iris_fine_fence_new(batch, IRIS_FENCE_BOTTOM_OF_PIPE); iris_fine_fence_reference(screen, &fence->fine[b], fine); iris_fine_fence_reference(screen, &fine, NULL); } else { /* This batch has no commands queued up (perhaps we just flushed, * or all the commands are on the other batch). Wait for the last * syncobj on this engine - unless it's already finished by now. */ if (iris_fine_fence_signaled(batch->last_fence)) continue; iris_fine_fence_reference(screen, &fence->fine[b], batch->last_fence); } } iris_fence_reference(ctx->screen, out_fence, NULL); *out_fence = fence; } static void iris_fence_await(struct pipe_context *ctx, struct pipe_fence_handle *fence) { struct iris_context *ice = (struct iris_context *)ctx; /* Unflushed fences from the same context are no-ops. */ if (ctx && ctx == fence->unflushed_ctx) return; /* XXX: We can't safely flush the other context, because it might be * bound to another thread, and poking at its internals wouldn't * be safe. In the future we should use MI_SEMAPHORE_WAIT and * block until the other job has been submitted, relying on * kernel timeslicing to preempt us until the other job is * actually flushed and the seqno finally passes. */ if (fence->unflushed_ctx) { pipe_debug_message(&ice->dbg, CONFORMANCE, "%s", "glWaitSync on unflushed fence from another context " "is unlikely to work without kernel 5.8+\n"); } for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) { struct iris_fine_fence *fine = fence->fine[i]; if (iris_fine_fence_signaled(fine)) continue; for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) { struct iris_batch *batch = &ice->batches[b]; /* We're going to make any future work in this batch wait for our * fence to have gone by. But any currently queued work doesn't * need to wait. Flush the batch now, so it can happen sooner. */ iris_batch_flush(batch); /* Before adding a new reference, clean out any stale ones. */ clear_stale_syncobjs(batch); iris_batch_add_syncobj(batch, fine->syncobj, I915_EXEC_FENCE_WAIT); } } } #define NSEC_PER_SEC (1000 * USEC_PER_SEC) #define USEC_PER_SEC (1000 * MSEC_PER_SEC) #define MSEC_PER_SEC (1000) static uint64_t gettime_ns(void) { struct timespec current; clock_gettime(CLOCK_MONOTONIC, ¤t); return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec; } static uint64_t rel2abs(uint64_t timeout) { if (timeout == 0) return 0; uint64_t current_time = gettime_ns(); uint64_t max_timeout = (uint64_t) INT64_MAX - current_time; timeout = MIN2(max_timeout, timeout); return current_time + timeout; } static bool iris_fence_finish(struct pipe_screen *p_screen, struct pipe_context *ctx, struct pipe_fence_handle *fence, uint64_t timeout) { ctx = threaded_context_unwrap_sync(ctx); struct iris_context *ice = (struct iris_context *)ctx; struct iris_screen *screen = (struct iris_screen *)p_screen; /* If we created the fence with PIPE_FLUSH_DEFERRED, we may not have * flushed yet. Check if our syncobj is the current batch's signalling * syncobj - if so, we haven't flushed and need to now. * * The Gallium docs mention that a flush will occur if \p ctx matches * the context the fence was created with. It may be NULL, so we check * that it matches first. */ if (ctx && ctx == fence->unflushed_ctx) { for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++) { struct iris_fine_fence *fine = fence->fine[i]; if (iris_fine_fence_signaled(fine)) continue; if (fine->syncobj == iris_batch_get_signal_syncobj(&ice->batches[i])) iris_batch_flush(&ice->batches[i]); } /* The fence is no longer deferred. */ fence->unflushed_ctx = NULL; } unsigned int handle_count = 0; uint32_t handles[ARRAY_SIZE(fence->fine)]; for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) { struct iris_fine_fence *fine = fence->fine[i]; if (iris_fine_fence_signaled(fine)) continue; handles[handle_count++] = fine->syncobj->handle; } if (handle_count == 0) return true; struct drm_syncobj_wait args = { .handles = (uintptr_t)handles, .count_handles = handle_count, .timeout_nsec = rel2abs(timeout), .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL }; if (fence->unflushed_ctx) { /* This fence had a deferred flush from another context. We can't * safely flush it here, because the context might be bound to a * different thread, and poking at its internals wouldn't be safe. * * Instead, use the WAIT_FOR_SUBMIT flag to block and hope that * another thread submits the work. */ args.flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT; } return intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0; } static int sync_merge_fd(int sync_fd, int new_fd) { if (sync_fd == -1) return new_fd; if (new_fd == -1) return sync_fd; struct sync_merge_data args = { .name = "iris fence", .fd2 = new_fd, .fence = -1, }; intel_ioctl(sync_fd, SYNC_IOC_MERGE, &args); close(new_fd); close(sync_fd); return args.fence; } static int iris_fence_get_fd(struct pipe_screen *p_screen, struct pipe_fence_handle *fence) { struct iris_screen *screen = (struct iris_screen *)p_screen; int fd = -1; /* Deferred fences aren't supported. */ if (fence->unflushed_ctx) return -1; for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) { struct iris_fine_fence *fine = fence->fine[i]; if (iris_fine_fence_signaled(fine)) continue; struct drm_syncobj_handle args = { .handle = fine->syncobj->handle, .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1, }; intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args); fd = sync_merge_fd(fd, args.fd); } if (fd == -1) { /* Our fence has no syncobj's recorded. This means that all of the * batches had already completed, their syncobj's had been signalled, * and so we didn't bother to record them. But we're being asked to * export such a fence. So export a dummy already-signalled syncobj. */ struct drm_syncobj_handle args = { .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1, }; args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED); intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args); gem_syncobj_destroy(screen->fd, args.handle); return args.fd; } return fd; } static void iris_fence_create_fd(struct pipe_context *ctx, struct pipe_fence_handle **out, int fd, enum pipe_fd_type type) { assert(type == PIPE_FD_TYPE_NATIVE_SYNC || type == PIPE_FD_TYPE_SYNCOBJ); struct iris_screen *screen = (struct iris_screen *)ctx->screen; struct drm_syncobj_handle args = { .fd = fd, }; if (type == PIPE_FD_TYPE_NATIVE_SYNC) { args.flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE; args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED); } if (intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) { fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n", strerror(errno)); if (type == PIPE_FD_TYPE_NATIVE_SYNC) gem_syncobj_destroy(screen->fd, args.handle); *out = NULL; return; } struct iris_syncobj *syncobj = malloc(sizeof(*syncobj)); if (!syncobj) { *out = NULL; return; } syncobj->handle = args.handle; pipe_reference_init(&syncobj->ref, 1); struct iris_fine_fence *fine = calloc(1, sizeof(*fine)); if (!fine) { free(syncobj); *out = NULL; return; } static const uint32_t zero = 0; /* Fences work in terms of iris_fine_fence, but we don't actually have a * seqno for an imported fence. So, create a fake one which always * returns as 'not signaled' so we fall back to using the sync object. */ fine->seqno = UINT32_MAX; fine->map = &zero; fine->syncobj = syncobj; fine->flags = IRIS_FENCE_END; pipe_reference_init(&fine->reference, 1); struct pipe_fence_handle *fence = calloc(1, sizeof(*fence)); if (!fence) { free(fine); free(syncobj); *out = NULL; return; } pipe_reference_init(&fence->ref, 1); fence->fine[0] = fine; *out = fence; } static void iris_fence_signal(struct pipe_context *ctx, struct pipe_fence_handle *fence) { struct iris_context *ice = (struct iris_context *)ctx; if (ctx == fence->unflushed_ctx) return; for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) { for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) { struct iris_fine_fence *fine = fence->fine[i]; /* already signaled fence skipped */ if (iris_fine_fence_signaled(fine)) continue; ice->batches[b].contains_fence_signal = true; iris_batch_add_syncobj(&ice->batches[b], fine->syncobj, I915_EXEC_FENCE_SIGNAL); } } } void iris_init_screen_fence_functions(struct pipe_screen *screen) { screen->fence_reference = iris_fence_reference; screen->fence_finish = iris_fence_finish; screen->fence_get_fd = iris_fence_get_fd; } void iris_init_context_fence_functions(struct pipe_context *ctx) { ctx->flush = iris_fence_flush; ctx->create_fence_fd = iris_fence_create_fd; ctx->fence_server_sync = iris_fence_await; ctx->fence_server_signal = iris_fence_signal; }