#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
- #include <linux/highmem.h>
#include <linux/fsnotify.h>
#include <linux/fadvise.h>
#include <linux/task_work.h>
#include "waitid.h"
#include "futex.h"
#include "napi.h"
+ #include "uring_cmd.h"
+ #include "memmap.h"
#include "timeout.h"
#include "poll.h"
.mode = 0644,
.proc_handler = proc_dointvec,
},
- {},
};
#endif
- static inline void io_submit_flush_completions(struct io_ring_ctx *ctx)
- {
- if (!wq_list_empty(&ctx->submit_state.compl_reqs) ||
- ctx->submit_state.cqes_count)
- __io_submit_flush_completions(ctx);
- }
-
static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
{
return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
fallback_work.work);
struct llist_node *node = llist_del_all(&ctx->fallback_llist);
struct io_kiocb *req, *tmp;
- struct io_tw_state ts = { .locked = true, };
+ struct io_tw_state ts = {};
percpu_ref_get(&ctx->refs);
mutex_lock(&ctx->uring_lock);
llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
req->io_task_work.func(req, &ts);
- if (WARN_ON_ONCE(!ts.locked))
- return;
io_submit_flush_completions(ctx);
mutex_unlock(&ctx->uring_lock);
percpu_ref_put(&ctx->refs);
{
struct io_ring_ctx *ctx;
int hash_bits;
+ bool ret;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
INIT_LIST_HEAD(&ctx->sqd_list);
INIT_LIST_HEAD(&ctx->cq_overflow_list);
INIT_LIST_HEAD(&ctx->io_buffers_cache);
- INIT_HLIST_HEAD(&ctx->io_buf_list);
- io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,
+ ret = io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,
sizeof(struct io_rsrc_node));
- io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,
+ ret |= io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
sizeof(struct async_poll));
- io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
+ ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
sizeof(struct io_async_msghdr));
- io_futex_cache_init(ctx);
+ ret |= io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,
+ sizeof(struct io_async_rw));
+ ret |= io_alloc_cache_init(&ctx->uring_cache, IO_ALLOC_CACHE_MAX,
+ sizeof(struct uring_cache));
+ ret |= io_futex_cache_init(ctx);
+ if (ret)
+ goto err;
init_completion(&ctx->ref_comp);
xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
mutex_init(&ctx->uring_lock);
init_llist_head(&ctx->work_llist);
INIT_LIST_HEAD(&ctx->tctx_list);
ctx->submit_state.free_list.next = NULL;
- INIT_WQ_LIST(&ctx->locked_free_list);
INIT_HLIST_HEAD(&ctx->waitid_list);
#ifdef CONFIG_FUTEX
INIT_HLIST_HEAD(&ctx->futex_list);
return ctx;
err:
+ io_alloc_cache_free(&ctx->rsrc_node_cache, kfree);
+ io_alloc_cache_free(&ctx->apoll_cache, kfree);
+ io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
+ io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
+ io_alloc_cache_free(&ctx->uring_cache, kfree);
+ io_futex_cache_free(ctx);
kfree(ctx->cancel_table.hbs);
kfree(ctx->cancel_table_locked.hbs);
xa_destroy(&ctx->io_bl_xa);
{
if (req->flags & REQ_F_BUFFER_SELECTED) {
spin_lock(&req->ctx->completion_lock);
- io_put_kbuf_comp(req);
+ io_kbuf_drop(req);
spin_unlock(&req->ctx->completion_lock);
}
/* don't serialize this request if the fs doesn't need it */
if (should_hash && (req->file->f_flags & O_DIRECT) &&
- (req->file->f_mode & FMODE_DIO_PARALLEL_WRITE))
+ (req->file->f_op->fop_flags & FOP_DIO_PARALLEL_WRITE))
should_hash = false;
if (should_hash || (ctx->flags & IORING_SETUP_IOPOLL))
io_wq_hash_work(&req->work, file_inode(req->file));
}
}
- void io_queue_iowq(struct io_kiocb *req, struct io_tw_state *ts_dont_use)
+ static void io_queue_iowq(struct io_kiocb *req)
{
struct io_kiocb *link = io_prep_linked_timeout(req);
struct io_uring_task *tctx = req->task->io_uring;
io_commit_cqring_flush(ctx);
}
- static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
- {
- struct io_overflow_cqe *ocqe;
- LIST_HEAD(list);
-
- spin_lock(&ctx->completion_lock);
- list_splice_init(&ctx->cq_overflow_list, &list);
- clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
- spin_unlock(&ctx->completion_lock);
-
- while (!list_empty(&list)) {
- ocqe = list_first_entry(&list, struct io_overflow_cqe, list);
- list_del(&ocqe->list);
- kfree(ocqe);
- }
- }
-
- static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)
+ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool dying)
{
size_t cqe_size = sizeof(struct io_uring_cqe);
- if (__io_cqring_events(ctx) == ctx->cq_entries)
+ lockdep_assert_held(&ctx->uring_lock);
+
+ /* don't abort if we're dying, entries must get freed */
+ if (!dying && __io_cqring_events(ctx) == ctx->cq_entries)
return;
if (ctx->flags & IORING_SETUP_CQE32)
struct io_uring_cqe *cqe;
struct io_overflow_cqe *ocqe;
- if (!io_get_cqe_overflow(ctx, &cqe, true))
- break;
ocqe = list_first_entry(&ctx->cq_overflow_list,
struct io_overflow_cqe, list);
- memcpy(cqe, &ocqe->cqe, cqe_size);
+
+ if (!dying) {
+ if (!io_get_cqe_overflow(ctx, &cqe, true))
+ break;
+ memcpy(cqe, &ocqe->cqe, cqe_size);
+ }
list_del(&ocqe->list);
kfree(ocqe);
}
io_cq_unlock_post(ctx);
}
- static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
+ static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
{
- /* iopoll syncs against uring_lock, not completion_lock */
- if (ctx->flags & IORING_SETUP_IOPOLL)
- mutex_lock(&ctx->uring_lock);
- __io_cqring_overflow_flush(ctx);
- if (ctx->flags & IORING_SETUP_IOPOLL)
- mutex_unlock(&ctx->uring_lock);
+ if (ctx->rings)
+ __io_cqring_overflow_flush(ctx, true);
}
- static void io_cqring_overflow_flush(struct io_ring_ctx *ctx)
+ static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
{
- if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
- io_cqring_do_overflow_flush(ctx);
+ mutex_lock(&ctx->uring_lock);
+ __io_cqring_overflow_flush(ctx, false);
+ mutex_unlock(&ctx->uring_lock);
}
/* can be called by any task */
return true;
}
- void io_req_cqe_overflow(struct io_kiocb *req)
+ static void io_req_cqe_overflow(struct io_kiocb *req)
{
io_cqring_event_overflow(req->ctx, req->cqe.user_data,
req->cqe.res, req->cqe.flags,
return false;
}
- static void __io_flush_post_cqes(struct io_ring_ctx *ctx)
- __must_hold(&ctx->uring_lock)
- {
- struct io_submit_state *state = &ctx->submit_state;
- unsigned int i;
-
- lockdep_assert_held(&ctx->uring_lock);
- for (i = 0; i < state->cqes_count; i++) {
- struct io_uring_cqe *cqe = &ctx->completion_cqes[i];
-
- if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) {
- if (ctx->lockless_cq) {
- spin_lock(&ctx->completion_lock);
- io_cqring_event_overflow(ctx, cqe->user_data,
- cqe->res, cqe->flags, 0, 0);
- spin_unlock(&ctx->completion_lock);
- } else {
- io_cqring_event_overflow(ctx, cqe->user_data,
- cqe->res, cqe->flags, 0, 0);
- }
- }
- }
- state->cqes_count = 0;
- }
-
- static bool __io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags,
- bool allow_overflow)
+ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
{
bool filled;
io_cq_lock(ctx);
filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
- if (!filled && allow_overflow)
+ if (!filled)
filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
io_cq_unlock_post(ctx);
return filled;
}
- bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
- {
- return __io_post_aux_cqe(ctx, user_data, res, cflags, true);
- }
-
/*
* A helper for multishot requests posting additional CQEs.
* Should only be used from a task_work including IO_URING_F_MULTISHOT.
*/
- bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags)
+ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)
{
struct io_ring_ctx *ctx = req->ctx;
- u64 user_data = req->cqe.user_data;
- struct io_uring_cqe *cqe;
+ bool posted;
lockdep_assert(!io_wq_current_is_worker());
-
- if (!defer)
- return __io_post_aux_cqe(ctx, user_data, res, cflags, false);
-
lockdep_assert_held(&ctx->uring_lock);
- if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->completion_cqes)) {
- __io_cq_lock(ctx);
- __io_flush_post_cqes(ctx);
- /* no need to flush - flush is deferred */
- __io_cq_unlock_post(ctx);
- }
-
- /* For defered completions this is not as strict as it is otherwise,
- * however it's main job is to prevent unbounded posted completions,
- * and in that it works just as well.
- */
- if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
- return false;
-
- cqe = &ctx->completion_cqes[ctx->submit_state.cqes_count++];
- cqe->user_data = user_data;
- cqe->res = res;
- cqe->flags = cflags;
- return true;
+ __io_cq_lock(ctx);
+ posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
+ ctx->submit_state.cq_flush = true;
+ __io_cq_unlock_post(ctx);
+ return posted;
}
- static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
+ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
{
struct io_ring_ctx *ctx = req->ctx;
- struct io_rsrc_node *rsrc_node = NULL;
+
+ /*
+ * All execution paths but io-wq use the deferred completions by
+ * passing IO_URING_F_COMPLETE_DEFER and thus should not end up here.
+ */
+ if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_IOWQ)))
+ return;
+
+ /*
+ * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
+ * the submitter task context, IOPOLL protects with uring_lock.
+ */
+ if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL)) {
+ req->io_task_work.func = io_req_task_complete;
+ io_req_task_work_add(req);
+ return;
+ }
io_cq_lock(ctx);
if (!(req->flags & REQ_F_CQE_SKIP)) {
if (!io_fill_cqe_req(ctx, req))
io_req_cqe_overflow(req);
}
+ io_cq_unlock_post(ctx);
/*
- * If we're the last reference to this request, add to our locked
- * free_list cache.
+ * We don't free the request here because we know it's called from
+ * io-wq only, which holds a reference, so it cannot be the last put.
*/
- if (req_ref_put_and_test(req)) {
- if (req->flags & IO_REQ_LINK_FLAGS) {
- if (req->flags & IO_DISARM_MASK)
- io_disarm_next(req);
- if (req->link) {
- io_req_task_queue(req->link);
- req->link = NULL;
- }
- }
- io_put_kbuf_comp(req);
- if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
- io_clean_op(req);
- io_put_file(req);
-
- rsrc_node = req->rsrc_node;
- /*
- * Selected buffer deallocation in io_clean_op() assumes that
- * we don't hold ->completion_lock. Clean them here to avoid
- * deadlocks.
- */
- io_put_task_remote(req->task);
- wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
- ctx->locked_free_nr++;
- }
- io_cq_unlock_post(ctx);
-
- if (rsrc_node) {
- io_ring_submit_lock(ctx, issue_flags);
- io_put_rsrc_node(ctx, rsrc_node);
- io_ring_submit_unlock(ctx, issue_flags);
- }
- }
-
- void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
- {
- struct io_ring_ctx *ctx = req->ctx;
-
- if (ctx->task_complete && ctx->submitter_task != current) {
- req->io_task_work.func = io_req_task_complete;
- io_req_task_work_add(req);
- } else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
- !(ctx->flags & IORING_SETUP_IOPOLL)) {
- __io_req_complete_post(req, issue_flags);
- } else {
- mutex_lock(&ctx->uring_lock);
- __io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);
- mutex_unlock(&ctx->uring_lock);
- }
+ req_ref_put(req);
}
void io_req_defer_failed(struct io_kiocb *req, s32 res)
memset(&req->big_cqe, 0, sizeof(req->big_cqe));
}
- static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
- struct io_submit_state *state)
- {
- spin_lock(&ctx->completion_lock);
- wq_list_splice(&ctx->locked_free_list, &state->free_list);
- ctx->locked_free_nr = 0;
- spin_unlock(&ctx->completion_lock);
- }
-
/*
* A request might get retired back into the request caches even before opcode
* handlers and io_issue_sqe() are done with it, e.g. inline completion path.
{
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
void *reqs[IO_REQ_ALLOC_BATCH];
- int ret, i;
-
- /*
- * If we have more than a batch's worth of requests in our IRQ side
- * locked cache, grab the lock and move them over to our submission
- * side cache.
- */
- if (data_race(ctx->locked_free_nr) > IO_COMPL_BATCH) {
- io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
- if (!io_req_cache_empty(ctx))
- return true;
- }
+ int ret;
ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
}
percpu_ref_get_many(&ctx->refs, ret);
- for (i = 0; i < ret; i++) {
- struct io_kiocb *req = reqs[i];
+ while (ret--) {
+ struct io_kiocb *req = reqs[ret];
io_preinit_req(req, ctx);
io_req_add_to_cache(req, ctx);
return;
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
- if (ts->locked) {
- io_submit_flush_completions(ctx);
- mutex_unlock(&ctx->uring_lock);
- ts->locked = false;
- }
+
+ io_submit_flush_completions(ctx);
+ mutex_unlock(&ctx->uring_lock);
percpu_ref_put(&ctx->refs);
}
if (req->ctx != ctx) {
ctx_flush_and_put(ctx, &ts);
ctx = req->ctx;
- /* if not contended, grab and improve batching */
- ts.locked = mutex_trylock(&ctx->uring_lock);
+ mutex_lock(&ctx->uring_lock);
percpu_ref_get(&ctx->refs);
}
INDIRECT_CALL_2(req->io_task_work.func,
if (io_run_local_work_continue(ctx, ret, min_events))
goto again;
- if (ts->locked) {
- io_submit_flush_completions(ctx);
- if (io_run_local_work_continue(ctx, ret, min_events))
- goto again;
- }
+ io_submit_flush_completions(ctx);
+ if (io_run_local_work_continue(ctx, ret, min_events))
+ goto again;
trace_io_uring_local_work_run(ctx, ret, loops);
return ret;
static inline int io_run_local_work_locked(struct io_ring_ctx *ctx,
int min_events)
{
- struct io_tw_state ts = { .locked = true, };
- int ret;
+ struct io_tw_state ts = {};
if (llist_empty(&ctx->work_llist))
return 0;
-
- ret = __io_run_local_work(ctx, &ts, min_events);
- /* shouldn't happen! */
- if (WARN_ON_ONCE(!ts.locked))
- mutex_lock(&ctx->uring_lock);
- return ret;
+ return __io_run_local_work(ctx, &ts, min_events);
}
static int io_run_local_work(struct io_ring_ctx *ctx, int min_events)
struct io_tw_state ts = {};
int ret;
- ts.locked = mutex_trylock(&ctx->uring_lock);
+ mutex_lock(&ctx->uring_lock);
ret = __io_run_local_work(ctx, &ts, min_events);
- if (ts.locked)
- mutex_unlock(&ctx->uring_lock);
-
+ mutex_unlock(&ctx->uring_lock);
return ret;
}
if (unlikely(req->task->flags & PF_EXITING))
io_req_defer_failed(req, -EFAULT);
else if (req->flags & REQ_F_FORCE_ASYNC)
- io_queue_iowq(req, ts);
+ io_queue_iowq(req);
else
io_queue_sqe(req);
}
if (apoll->double_poll)
kfree(apoll->double_poll);
- if (!io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache))
+ if (!io_alloc_cache_put(&ctx->apoll_cache, apoll))
kfree(apoll);
req->flags &= ~REQ_F_POLLED;
}
io_clean_op(req);
}
io_put_file(req);
-
- io_req_put_rsrc_locked(req, ctx);
-
+ io_put_rsrc_node(ctx, req->rsrc_node);
io_put_task(req->task);
+
node = req->comp_list.next;
io_req_add_to_cache(req, ctx);
} while (node);
struct io_wq_work_node *node;
__io_cq_lock(ctx);
- /* must come first to preserve CQE ordering in failure cases */
- if (state->cqes_count)
- __io_flush_post_cqes(ctx);
__wq_list_for_each(node, &state->compl_reqs) {
struct io_kiocb *req = container_of(node, struct io_kiocb,
comp_list);
io_free_batch_list(ctx, state->compl_reqs.first);
INIT_WQ_LIST(&state->compl_reqs);
}
+ ctx->submit_state.cq_flush = false;
}
static unsigned io_cqring_events(struct io_ring_ctx *ctx)
unsigned int nr_events = 0;
unsigned long check_cq;
+ lockdep_assert_held(&ctx->uring_lock);
+
if (!io_allowed_run_tw(ctx))
return -EEXIST;
check_cq = READ_ONCE(ctx->check_cq);
if (unlikely(check_cq)) {
if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
- __io_cqring_overflow_flush(ctx);
+ __io_cqring_overflow_flush(ctx, false);
/*
* Similarly do not spin if we have not informed the user of any
* dropped CQE.
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
{
- if (ts->locked)
- io_req_complete_defer(req);
- else
- io_req_complete_post(req, IO_URING_F_UNLOCKED);
+ io_req_complete_defer(req);
}
/*
bool io_alloc_async_data(struct io_kiocb *req)
{
- WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size);
- req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL);
+ const struct io_issue_def *def = &io_issue_defs[req->opcode];
+
+ WARN_ON_ONCE(!def->async_size);
+ req->async_data = kmalloc(def->async_size, GFP_KERNEL);
if (req->async_data) {
req->flags |= REQ_F_ASYNC_DATA;
return false;
return true;
}
- int io_req_prep_async(struct io_kiocb *req)
- {
- const struct io_cold_def *cdef = &io_cold_defs[req->opcode];
- const struct io_issue_def *def = &io_issue_defs[req->opcode];
-
- /* assign early for deferred execution for non-fixed file */
- if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE) && !req->file)
- req->file = io_file_get_normal(req, req->cqe.fd);
- if (!cdef->prep_async)
- return 0;
- if (WARN_ON_ONCE(req_has_async_data(req)))
- return -EFAULT;
- if (!def->manual_alloc) {
- if (io_alloc_async_data(req))
- return -EAGAIN;
- }
- return cdef->prep_async(req);
- }
-
static u32 io_get_sequence(struct io_kiocb *req)
{
u32 seq = req->ctx->cached_sq_head;
break;
case IO_APOLL_ABORTED:
io_kbuf_recycle(req, 0);
- io_queue_iowq(req, NULL);
+ io_queue_iowq(req);
break;
case IO_APOLL_OK:
break;
req->flags |= REQ_F_LINK;
io_req_defer_failed(req, req->cqe.res);
} else {
- int ret = io_req_prep_async(req);
-
- if (unlikely(ret)) {
- io_req_defer_failed(req, ret);
- return;
- }
-
if (unlikely(req->ctx->drain_active))
io_drain_req(req);
else
- io_queue_iowq(req, NULL);
+ io_queue_iowq(req);
}
}
* conditions are true (normal request), then just queue it.
*/
if (unlikely(link->head)) {
- ret = io_req_prep_async(req);
- if (unlikely(ret))
- return io_submit_fail_init(sqe, req, ret);
-
trace_io_uring_link(req, link->head);
link->last->link = req;
link->last = req;
if (!llist_empty(&ctx->work_llist))
io_run_local_work(ctx, min_events);
io_run_task_work();
- io_cqring_overflow_flush(ctx);
- /* if user messes with these they will just get an early return */
+
+ if (unlikely(test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)))
+ io_cqring_do_overflow_flush(ctx);
if (__io_cqring_events_user(ctx) >= min_events)
return 0;
return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
}
- void io_mem_free(void *ptr)
- {
- if (!ptr)
- return;
-
- folio_put(virt_to_folio(ptr));
- }
-
- static void io_pages_free(struct page ***pages, int npages)
- {
- struct page **page_array = *pages;
- int i;
-
- if (!page_array)
- return;
-
- for (i = 0; i < npages; i++)
- unpin_user_page(page_array[i]);
- kvfree(page_array);
- *pages = NULL;
- }
-
- static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
- unsigned long uaddr, size_t size)
- {
- struct page **page_array;
- unsigned int nr_pages;
- void *page_addr;
- int ret, i, pinned;
-
- *npages = 0;
-
- if (uaddr & (PAGE_SIZE - 1) || !size)
- return ERR_PTR(-EINVAL);
-
- nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
- if (nr_pages > USHRT_MAX)
- return ERR_PTR(-EINVAL);
- page_array = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
- if (!page_array)
- return ERR_PTR(-ENOMEM);
-
-
- pinned = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
- page_array);
- if (pinned != nr_pages) {
- ret = (pinned < 0) ? pinned : -EFAULT;
- goto free_pages;
- }
-
- page_addr = page_address(page_array[0]);
- for (i = 0; i < nr_pages; i++) {
- ret = -EINVAL;
-
- /*
- * Can't support mapping user allocated ring memory on 32-bit
- * archs where it could potentially reside in highmem. Just
- * fail those with -EINVAL, just like we did on kernels that
- * didn't support this feature.
- */
- if (PageHighMem(page_array[i]))
- goto free_pages;
-
- /*
- * No support for discontig pages for now, should either be a
- * single normal page, or a huge page. Later on we can add
- * support for remapping discontig pages, for now we will
- * just fail them with EINVAL.
- */
- if (page_address(page_array[i]) != page_addr)
- goto free_pages;
- page_addr += PAGE_SIZE;
- }
-
- *pages = page_array;
- *npages = nr_pages;
- return page_to_virt(page_array[0]);
-
- free_pages:
- io_pages_free(&page_array, pinned > 0 ? pinned : 0);
- return ERR_PTR(ret);
- }
-
static void *io_rings_map(struct io_ring_ctx *ctx, unsigned long uaddr,
size_t size)
{
static void io_rings_free(struct io_ring_ctx *ctx)
{
if (!(ctx->flags & IORING_SETUP_NO_MMAP)) {
- io_mem_free(ctx->rings);
- io_mem_free(ctx->sq_sqes);
+ io_pages_unmap(ctx->rings, &ctx->ring_pages, &ctx->n_ring_pages,
+ true);
+ io_pages_unmap(ctx->sq_sqes, &ctx->sqe_pages, &ctx->n_sqe_pages,
+ true);
} else {
io_pages_free(&ctx->ring_pages, ctx->n_ring_pages);
ctx->n_ring_pages = 0;
io_pages_free(&ctx->sqe_pages, ctx->n_sqe_pages);
ctx->n_sqe_pages = 0;
+ vunmap(ctx->rings);
+ vunmap(ctx->sq_sqes);
}
ctx->rings = NULL;
ctx->sq_sqes = NULL;
}
- void *io_mem_alloc(size_t size)
- {
- gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
- void *ret;
-
- ret = (void *) __get_free_pages(gfp, get_order(size));
- if (ret)
- return ret;
- return ERR_PTR(-ENOMEM);
- }
-
static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset)
{
int nr = 0;
mutex_lock(&ctx->uring_lock);
- io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
while (!io_req_cache_empty(ctx)) {
req = io_extract_req(ctx);
mutex_unlock(&ctx->uring_lock);
}
- static void io_rsrc_node_cache_free(struct io_cache_entry *entry)
- {
- kfree(container_of(entry, struct io_rsrc_node, cache));
- }
-
static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
io_sq_thread_finish(ctx);
__io_sqe_files_unregister(ctx);
io_cqring_overflow_kill(ctx);
io_eventfd_unregister(ctx);
- io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);
+ io_alloc_cache_free(&ctx->apoll_cache, kfree);
io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
+ io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
+ io_alloc_cache_free(&ctx->uring_cache, kfree);
io_futex_cache_free(ctx);
io_destroy_buffers(ctx);
mutex_unlock(&ctx->uring_lock);
WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
- io_alloc_cache_free(&ctx->rsrc_node_cache, io_rsrc_node_cache_free);
+ io_alloc_cache_free(&ctx->rsrc_node_cache, kfree);
if (ctx->mm_account) {
mmdrop(ctx->mm_account);
ctx->mm_account = NULL;
}
io_rings_free(ctx);
- io_kbuf_mmap_list_free(ctx);
percpu_ref_exit(&ctx->refs);
free_uid(ctx->user);
percpu_ref_kill(&ctx->refs);
xa_for_each(&ctx->personalities, index, creds)
io_unregister_personality(ctx, index);
- if (ctx->rings)
- io_poll_remove_all(ctx, NULL, true);
mutex_unlock(&ctx->uring_lock);
- /*
- * If we failed setting up the ctx, we might not have any rings
- * and therefore did not submit any requests
- */
- if (ctx->rings)
- io_kill_timeouts(ctx, NULL, true);
-
flush_delayed_work(&ctx->fallback_work);
INIT_WORK(&ctx->exit_work, io_ring_exit_work);
return ret;
}
- static bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
- struct task_struct *task, bool cancel_all)
- {
- struct hlist_node *tmp;
- struct io_kiocb *req;
- bool ret = false;
-
- lockdep_assert_held(&ctx->uring_lock);
-
- hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
- hash_node) {
- struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
- struct io_uring_cmd);
- struct file *file = req->file;
-
- if (!cancel_all && req->task != task)
- continue;
-
- if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
- /* ->sqe isn't available if no async data */
- if (!req_has_async_data(req))
- cmd->sqe = NULL;
- file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL);
- ret = true;
- }
- }
- io_submit_flush_completions(ctx);
-
- return ret;
- }
-
static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
struct task_struct *task,
bool cancel_all)
ret |= io_kill_timeouts(ctx, task, cancel_all);
if (task)
ret |= io_run_task_work() > 0;
+ else
+ ret |= flush_delayed_work(&ctx->fallback_work);
return ret;
}
io_uring_cancel_generic(cancel_all, NULL);
}
- static void *io_uring_validate_mmap_request(struct file *file,
- loff_t pgoff, size_t sz)
- {
- struct io_ring_ctx *ctx = file->private_data;
- loff_t offset = pgoff << PAGE_SHIFT;
- struct page *page;
- void *ptr;
-
- switch (offset & IORING_OFF_MMAP_MASK) {
- case IORING_OFF_SQ_RING:
- case IORING_OFF_CQ_RING:
- /* Don't allow mmap if the ring was setup without it */
- if (ctx->flags & IORING_SETUP_NO_MMAP)
- return ERR_PTR(-EINVAL);
- ptr = ctx->rings;
- break;
- case IORING_OFF_SQES:
- /* Don't allow mmap if the ring was setup without it */
- if (ctx->flags & IORING_SETUP_NO_MMAP)
- return ERR_PTR(-EINVAL);
- ptr = ctx->sq_sqes;
- break;
- case IORING_OFF_PBUF_RING: {
- struct io_buffer_list *bl;
- unsigned int bgid;
-
- bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
- bl = io_pbuf_get_bl(ctx, bgid);
- if (IS_ERR(bl))
- return bl;
- ptr = bl->buf_ring;
- io_put_bl(ctx, bl);
- break;
- }
- default:
- return ERR_PTR(-EINVAL);
- }
-
- page = virt_to_head_page(ptr);
- if (sz > page_size(page))
- return ERR_PTR(-EINVAL);
-
- return ptr;
- }
-
- #ifdef CONFIG_MMU
-
- static __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
- {
- size_t sz = vma->vm_end - vma->vm_start;
- unsigned long pfn;
- void *ptr;
-
- ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
- if (IS_ERR(ptr))
- return PTR_ERR(ptr);
-
- pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
- return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
- }
-
- static unsigned long io_uring_mmu_get_unmapped_area(struct file *filp,
- unsigned long addr, unsigned long len,
- unsigned long pgoff, unsigned long flags)
- {
- void *ptr;
-
- /*
- * Do not allow to map to user-provided address to avoid breaking the
- * aliasing rules. Userspace is not able to guess the offset address of
- * kernel kmalloc()ed memory area.
- */
- if (addr)
- return -EINVAL;
-
- ptr = io_uring_validate_mmap_request(filp, pgoff, len);
- if (IS_ERR(ptr))
- return -ENOMEM;
-
- /*
- * Some architectures have strong cache aliasing requirements.
- * For such architectures we need a coherent mapping which aliases
- * kernel memory *and* userspace memory. To achieve that:
- * - use a NULL file pointer to reference physical memory, and
- * - use the kernel virtual address of the shared io_uring context
- * (instead of the userspace-provided address, which has to be 0UL
- * anyway).
- * - use the same pgoff which the get_unmapped_area() uses to
- * calculate the page colouring.
- * For architectures without such aliasing requirements, the
- * architecture will return any suitable mapping because addr is 0.
- */
- filp = NULL;
- flags |= MAP_SHARED;
- pgoff = 0; /* has been translated to ptr above */
- #ifdef SHM_COLOUR
- addr = (uintptr_t) ptr;
- pgoff = addr >> PAGE_SHIFT;
- #else
- addr = 0UL;
- #endif
- return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
- }
-
- #else /* !CONFIG_MMU */
-
- static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
- {
- return is_nommu_shared_mapping(vma->vm_flags) ? 0 : -EINVAL;
- }
-
- static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
- {
- return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
- }
-
- static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
- unsigned long addr, unsigned long len,
- unsigned long pgoff, unsigned long flags)
- {
- void *ptr;
-
- ptr = io_uring_validate_mmap_request(file, pgoff, len);
- if (IS_ERR(ptr))
- return PTR_ERR(ptr);
-
- return (unsigned long) ptr;
- }
-
- #endif /* !CONFIG_MMU */
-
static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t argsz)
{
if (flags & IORING_ENTER_EXT_ARG) {
*/
ret = 0;
if (ctx->flags & IORING_SETUP_SQPOLL) {
- io_cqring_overflow_flush(ctx);
-
if (unlikely(ctx->sq_data->thread == NULL)) {
ret = -EOWNERDEAD;
goto out;
static const struct file_operations io_uring_fops = {
.release = io_uring_release,
.mmap = io_uring_mmap,
+ .get_unmapped_area = io_uring_get_unmapped_area,
#ifndef CONFIG_MMU
- .get_unmapped_area = io_uring_nommu_get_unmapped_area,
.mmap_capabilities = io_uring_nommu_mmap_capabilities,
- #else
- .get_unmapped_area = io_uring_mmu_get_unmapped_area,
#endif
.poll = io_uring_poll,
#ifdef CONFIG_PROC_FS
return -EOVERFLOW;
if (!(ctx->flags & IORING_SETUP_NO_MMAP))
- rings = io_mem_alloc(size);
+ rings = io_pages_map(&ctx->ring_pages, &ctx->n_ring_pages, size);
else
rings = io_rings_map(ctx, p->cq_off.user_addr, size);
}
if (!(ctx->flags & IORING_SETUP_NO_MMAP))
- ptr = io_mem_alloc(size);
+ ptr = io_pages_map(&ctx->sqe_pages, &ctx->n_sqe_pages, size);
else
ptr = io_sqes_map(ctx, p->sq_off.user_addr, size);
IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP |
- IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING;
+ IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING |
+ IORING_FEAT_RECVSEND_BUNDLE;
if (copy_to_user(params, p, sizeof(*p))) {
ret = -EFAULT;
#include "io_uring.h"
#include "opdef.h"
#include "kbuf.h"
+ #include "alloc_cache.h"
#include "rsrc.h"
#include "poll.h"
#include "rw.h"
return 0;
}
- int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ static int __io_import_iovec(int ddir, struct io_kiocb *req,
+ struct io_async_rw *io,
+ unsigned int issue_flags)
+ {
+ const struct io_issue_def *def = &io_issue_defs[req->opcode];
+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+ struct iovec *iov;
+ void __user *buf;
+ int nr_segs, ret;
+ size_t sqe_len;
+
+ buf = u64_to_user_ptr(rw->addr);
+ sqe_len = rw->len;
+
+ if (!def->vectored || req->flags & REQ_F_BUFFER_SELECT) {
+ if (io_do_buffer_select(req)) {
+ buf = io_buffer_select(req, &sqe_len, issue_flags);
+ if (!buf)
+ return -ENOBUFS;
+ rw->addr = (unsigned long) buf;
+ rw->len = sqe_len;
+ }
+
+ return import_ubuf(ddir, buf, sqe_len, &io->iter);
+ }
+
+ if (io->free_iovec) {
+ nr_segs = io->free_iov_nr;
+ iov = io->free_iovec;
+ } else {
+ iov = &io->fast_iov;
+ nr_segs = 1;
+ }
+ ret = __import_iovec(ddir, buf, sqe_len, nr_segs, &iov, &io->iter,
+ req->ctx->compat);
+ if (unlikely(ret < 0))
+ return ret;
+ if (iov) {
+ req->flags |= REQ_F_NEED_CLEANUP;
+ io->free_iov_nr = io->iter.nr_segs;
+ kfree(io->free_iovec);
+ io->free_iovec = iov;
+ }
+ return 0;
+ }
+
+ static inline int io_import_iovec(int rw, struct io_kiocb *req,
+ struct io_async_rw *io,
+ unsigned int issue_flags)
+ {
+ int ret;
+
+ ret = __io_import_iovec(rw, req, io, issue_flags);
+ if (unlikely(ret < 0))
+ return ret;
+
+ iov_iter_save_state(&io->iter, &io->iter_state);
+ return 0;
+ }
+
+ static void io_rw_iovec_free(struct io_async_rw *rw)
+ {
+ if (rw->free_iovec) {
+ kfree(rw->free_iovec);
+ rw->free_iov_nr = 0;
+ rw->free_iovec = NULL;
+ }
+ }
+
+ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)
+ {
+ struct io_async_rw *rw = req->async_data;
+ struct iovec *iov;
+
+ if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
+ io_rw_iovec_free(rw);
+ return;
+ }
+ iov = rw->free_iovec;
+ if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) {
+ if (iov)
+ kasan_mempool_poison_object(iov);
+ req->async_data = NULL;
+ req->flags &= ~REQ_F_ASYNC_DATA;
+ }
+ }
+
+ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags)
+ {
+ /*
+ * Disable quick recycling for anything that's gone through io-wq.
+ * In theory, this should be fine to cleanup. However, some read or
+ * write iter handling touches the iovec AFTER having called into the
+ * handler, eg to reexpand or revert. This means we can have:
+ *
+ * task io-wq
+ * issue
+ * punt to io-wq
+ * issue
+ * blkdev_write_iter()
+ * ->ki_complete()
+ * io_complete_rw()
+ * queue tw complete
+ * run tw
+ * req_rw_cleanup
+ * iov_iter_count() <- look at iov_iter again
+ *
+ * which can lead to a UAF. This is only possible for io-wq offload
+ * as the cleanup can run in parallel. As io-wq is not the fast path,
+ * just leave cleanup to the end.
+ *
+ * This is really a bug in the core code that does this, any issue
+ * path should assume that a successful (or -EIOCBQUEUED) return can
+ * mean that the underlying data can be gone at any time. But that
+ * should be fixed seperately, and then this check could be killed.
+ */
+ if (!(req->flags & REQ_F_REFCOUNT)) {
+ req->flags &= ~REQ_F_NEED_CLEANUP;
+ io_rw_recycle(req, issue_flags);
+ }
+ }
+
+ static int io_rw_alloc_async(struct io_kiocb *req)
+ {
+ struct io_ring_ctx *ctx = req->ctx;
+ struct io_async_rw *rw;
+
+ rw = io_alloc_cache_get(&ctx->rw_cache);
+ if (rw) {
+ if (rw->free_iovec) {
+ kasan_mempool_unpoison_object(rw->free_iovec,
+ rw->free_iov_nr * sizeof(struct iovec));
+ req->flags |= REQ_F_NEED_CLEANUP;
+ }
+ req->flags |= REQ_F_ASYNC_DATA;
+ req->async_data = rw;
+ goto done;
+ }
+
+ if (!io_alloc_async_data(req)) {
+ rw = req->async_data;
+ rw->free_iovec = NULL;
+ rw->free_iov_nr = 0;
+ done:
+ rw->bytes_done = 0;
+ return 0;
+ }
+
+ return -ENOMEM;
+ }
+
+ static int io_prep_rw_setup(struct io_kiocb *req, int ddir, bool do_import)
+ {
+ struct io_async_rw *rw;
+ int ret;
+
+ if (io_rw_alloc_async(req))
+ return -ENOMEM;
+
+ if (!do_import || io_do_buffer_select(req))
+ return 0;
+
+ rw = req->async_data;
+ ret = io_import_iovec(ddir, req, rw, 0);
+ if (unlikely(ret < 0))
+ return ret;
+
+ iov_iter_save_state(&rw->iter, &rw->iter_state);
+ return 0;
+ }
+
+ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ int ddir, bool do_import)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
unsigned ioprio;
rw->addr = READ_ONCE(sqe->addr);
rw->len = READ_ONCE(sqe->len);
rw->flags = READ_ONCE(sqe->rw_flags);
- return 0;
+ return io_prep_rw_setup(req, ddir, do_import);
+ }
+
+ int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ return io_prep_rw(req, sqe, ITER_DEST, true);
}
- int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
+ return io_prep_rw(req, sqe, ITER_SOURCE, true);
+ }
+
+ static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ int ddir)
+ {
+ const bool do_import = !(req->flags & REQ_F_BUFFER_SELECT);
int ret;
- ret = io_prep_rw(req, sqe);
+ ret = io_prep_rw(req, sqe, ddir, do_import);
if (unlikely(ret))
return ret;
+ if (do_import)
+ return 0;
/*
* Have to do this validation here, as this is in io_read() rw->len
* might have chanaged due to buffer selection
*/
- if (req->flags & REQ_F_BUFFER_SELECT)
- return io_iov_buffer_select_prep(req);
+ return io_iov_buffer_select_prep(req);
+ }
- return 0;
+ int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ return io_prep_rwv(req, sqe, ITER_DEST);
+ }
+
+ int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ return io_prep_rwv(req, sqe, ITER_SOURCE);
}
- int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ int ddir)
{
+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
struct io_ring_ctx *ctx = req->ctx;
+ struct io_async_rw *io;
u16 index;
int ret;
- ret = io_prep_rw(req, sqe);
+ ret = io_prep_rw(req, sqe, ddir, false);
if (unlikely(ret))
return ret;
index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
req->imu = ctx->user_bufs[index];
io_req_set_rsrc_node(req, ctx, 0);
- return 0;
+
+ io = req->async_data;
+ ret = io_import_fixed(ddir, &io->iter, req->imu, rw->addr, rw->len);
+ iov_iter_save_state(&io->iter, &io->iter_state);
+ return ret;
+ }
+
+ int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ return io_prep_rw_fixed(req, sqe, ITER_DEST);
+ }
+
+ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ return io_prep_rw_fixed(req, sqe, ITER_SOURCE);
}
/*
if (!(req->flags & REQ_F_BUFFER_SELECT))
return -EINVAL;
- ret = io_prep_rw(req, sqe);
+ ret = io_prep_rw(req, sqe, ITER_DEST, false);
if (unlikely(ret))
return ret;
void io_readv_writev_cleanup(struct io_kiocb *req)
{
- struct io_async_rw *io = req->async_data;
-
- kfree(io->free_iovec);
+ io_rw_iovec_free(req->async_data);
}
static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req)
return NULL;
}
- static void io_req_task_queue_reissue(struct io_kiocb *req)
- {
- req->io_task_work.func = io_queue_iowq;
- io_req_task_work_add(req);
- }
-
#ifdef CONFIG_BLOCK
- static bool io_resubmit_prep(struct io_kiocb *req)
+ static void io_resubmit_prep(struct io_kiocb *req)
{
struct io_async_rw *io = req->async_data;
- if (!req_has_async_data(req))
- return !io_req_prep_async(req);
- iov_iter_restore(&io->s.iter, &io->s.iter_state);
- return true;
+ iov_iter_restore(&io->iter, &io->iter_state);
}
static bool io_rw_should_reissue(struct io_kiocb *req)
return true;
}
#else
- static bool io_resubmit_prep(struct io_kiocb *req)
+ static void io_resubmit_prep(struct io_kiocb *req)
{
- return false;
}
static bool io_rw_should_reissue(struct io_kiocb *req)
{
io_req_io_end(req);
- if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) {
- unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED;
+ if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))
+ req->cqe.flags |= io_put_kbuf(req, 0);
- req->cqe.flags |= io_put_kbuf(req, issue_flags);
- }
+ io_req_rw_cleanup(req, 0);
io_req_task_complete(req, ts);
}
io_req_io_end(req);
io_req_set_res(req, final_ret,
io_put_kbuf(req, issue_flags));
+ io_req_rw_cleanup(req, issue_flags);
return IOU_OK;
}
} else {
if (req->flags & REQ_F_REISSUE) {
req->flags &= ~REQ_F_REISSUE;
- if (io_resubmit_prep(req))
- io_req_task_queue_reissue(req);
- else
- io_req_task_queue_fail(req, final_ret);
+ io_resubmit_prep(req);
+ return -EAGAIN;
}
return IOU_ISSUE_SKIP_COMPLETE;
}
- static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,
- struct io_rw_state *s,
- unsigned int issue_flags)
- {
- struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
- struct iov_iter *iter = &s->iter;
- u8 opcode = req->opcode;
- struct iovec *iovec;
- void __user *buf;
- size_t sqe_len;
- ssize_t ret;
-
- if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
- ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len);
- if (ret)
- return ERR_PTR(ret);
- return NULL;
- }
-
- buf = u64_to_user_ptr(rw->addr);
- sqe_len = rw->len;
-
- if (!io_issue_defs[opcode].vectored || req->flags & REQ_F_BUFFER_SELECT) {
- if (io_do_buffer_select(req)) {
- buf = io_buffer_select(req, &sqe_len, issue_flags);
- if (!buf)
- return ERR_PTR(-ENOBUFS);
- rw->addr = (unsigned long) buf;
- rw->len = sqe_len;
- }
-
- ret = import_ubuf(ddir, buf, sqe_len, iter);
- if (ret)
- return ERR_PTR(ret);
- return NULL;
- }
-
- iovec = s->fast_iov;
- ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter,
- req->ctx->compat);
- if (unlikely(ret < 0))
- return ERR_PTR(ret);
- return iovec;
- }
-
- static inline int io_import_iovec(int rw, struct io_kiocb *req,
- struct iovec **iovec, struct io_rw_state *s,
- unsigned int issue_flags)
- {
- *iovec = __io_import_iovec(rw, req, s, issue_flags);
- if (IS_ERR(*iovec))
- return PTR_ERR(*iovec);
-
- iov_iter_save_state(&s->iter, &s->iter_state);
- return 0;
- }
-
static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
{
return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
return ret;
}
- static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
- const struct iovec *fast_iov, struct iov_iter *iter)
- {
- struct io_async_rw *io = req->async_data;
-
- memcpy(&io->s.iter, iter, sizeof(*iter));
- io->free_iovec = iovec;
- io->bytes_done = 0;
- /* can only be fixed buffers, no need to do anything */
- if (iov_iter_is_bvec(iter) || iter_is_ubuf(iter))
- return;
- if (!iovec) {
- unsigned iov_off = 0;
-
- io->s.iter.__iov = io->s.fast_iov;
- if (iter->__iov != fast_iov) {
- iov_off = iter_iov(iter) - fast_iov;
- io->s.iter.__iov += iov_off;
- }
- if (io->s.fast_iov != fast_iov)
- memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off,
- sizeof(struct iovec) * iter->nr_segs);
- } else {
- req->flags |= REQ_F_NEED_CLEANUP;
- }
- }
-
- static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
- struct io_rw_state *s, bool force)
- {
- if (!force && !io_cold_defs[req->opcode].prep_async)
- return 0;
- /* opcode type doesn't need async data */
- if (!io_cold_defs[req->opcode].async_size)
- return 0;
- if (!req_has_async_data(req)) {
- struct io_async_rw *iorw;
-
- if (io_alloc_async_data(req)) {
- kfree(iovec);
- return -ENOMEM;
- }
-
- io_req_map_rw(req, iovec, s->fast_iov, &s->iter);
- iorw = req->async_data;
- /* we've copied and mapped the iter, ensure state is saved */
- iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state);
- }
- return 0;
- }
-
- static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
- {
- struct io_async_rw *iorw = req->async_data;
- struct iovec *iov;
- int ret;
-
- iorw->bytes_done = 0;
- iorw->free_iovec = NULL;
-
- /* submission path, ->uring_lock should already be taken */
- ret = io_import_iovec(rw, req, &iov, &iorw->s, 0);
- if (unlikely(ret < 0))
- return ret;
-
- if (iov) {
- iorw->free_iovec = iov;
- req->flags |= REQ_F_NEED_CLEANUP;
- }
-
- return 0;
- }
-
- int io_readv_prep_async(struct io_kiocb *req)
- {
- return io_rw_prep_async(req, ITER_DEST);
- }
-
- int io_writev_prep_async(struct io_kiocb *req)
- {
- return io_rw_prep_async(req, ITER_SOURCE);
- }
-
/*
* This is our waitqueue callback handler, registered through __folio_lock_async()
* when we initially tried to do the IO with the iocb armed our waitqueue.
* just use poll if we can, and don't attempt if the fs doesn't
* support callback based unlocks
*/
- if (io_file_can_poll(req) || !(req->file->f_mode & FMODE_BUF_RASYNC))
+ if (io_file_can_poll(req) ||
+ !(req->file->f_op->fop_flags & FOP_BUFFER_RASYNC))
return false;
wait->wait.func = io_async_buf_func;
static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
{
+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
- struct io_rw_state __s, *s = &__s;
- struct iovec *iovec;
+ struct io_async_rw *io = req->async_data;
struct kiocb *kiocb = &rw->kiocb;
- bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
- struct io_async_rw *io;
- ssize_t ret, ret2;
+ ssize_t ret;
loff_t *ppos;
- if (!req_has_async_data(req)) {
- ret = io_import_iovec(ITER_DEST, req, &iovec, s, issue_flags);
+ if (io_do_buffer_select(req)) {
+ ret = io_import_iovec(ITER_DEST, req, io, issue_flags);
if (unlikely(ret < 0))
return ret;
- } else {
- io = req->async_data;
- s = &io->s;
-
- /*
- * Safe and required to re-import if we're using provided
- * buffers, as we dropped the selected one before retry.
- */
- if (io_do_buffer_select(req)) {
- ret = io_import_iovec(ITER_DEST, req, &iovec, s, issue_flags);
- if (unlikely(ret < 0))
- return ret;
- }
-
- /*
- * We come here from an earlier attempt, restore our state to
- * match in case it doesn't. It's cheap enough that we don't
- * need to make this conditional.
- */
- iov_iter_restore(&s->iter, &s->iter_state);
- iovec = NULL;
}
+
ret = io_rw_init_file(req, FMODE_READ);
- if (unlikely(ret)) {
- kfree(iovec);
+ if (unlikely(ret))
return ret;
- }
- req->cqe.res = iov_iter_count(&s->iter);
+ req->cqe.res = iov_iter_count(&io->iter);
if (force_nonblock) {
/* If the file doesn't support async, just async punt */
- if (unlikely(!io_file_supports_nowait(req))) {
- ret = io_setup_async_rw(req, iovec, s, true);
- return ret ?: -EAGAIN;
- }
+ if (unlikely(!io_file_supports_nowait(req)))
+ return -EAGAIN;
kiocb->ki_flags |= IOCB_NOWAIT;
} else {
/* Ensure we clear previously set non-block flag */
ppos = io_kiocb_update_pos(req);
ret = rw_verify_area(READ, req->file, ppos, req->cqe.res);
- if (unlikely(ret)) {
- kfree(iovec);
+ if (unlikely(ret))
return ret;
- }
- ret = io_iter_do_read(rw, &s->iter);
+ ret = io_iter_do_read(rw, &io->iter);
if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
req->flags &= ~REQ_F_REISSUE;
- /*
- * If we can poll, just do that. For a vectored read, we'll
- * need to copy state first.
- */
- if (io_file_can_poll(req) && !io_issue_defs[req->opcode].vectored)
+ /* If we can poll, just do that. */
+ if (io_file_can_poll(req))
return -EAGAIN;
/* IOPOLL retry should happen for io-wq threads */
if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
goto done;
ret = 0;
} else if (ret == -EIOCBQUEUED) {
- if (iovec)
- kfree(iovec);
return IOU_ISSUE_SKIP_COMPLETE;
} else if (ret == req->cqe.res || ret <= 0 || !force_nonblock ||
(req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) {
* untouched in case of error. Restore it and we'll advance it
* manually if we need to.
*/
- iov_iter_restore(&s->iter, &s->iter_state);
-
- ret2 = io_setup_async_rw(req, iovec, s, true);
- iovec = NULL;
- if (ret2) {
- ret = ret > 0 ? ret : ret2;
- goto done;
- }
-
- io = req->async_data;
- s = &io->s;
- /*
- * Now use our persistent iterator and state, if we aren't already.
- * We've restored and mapped the iter to match.
- */
+ iov_iter_restore(&io->iter, &io->iter_state);
do {
/*
* above or inside this loop. Advance the iter by the bytes
* that were consumed.
*/
- iov_iter_advance(&s->iter, ret);
- if (!iov_iter_count(&s->iter))
+ iov_iter_advance(&io->iter, ret);
+ if (!iov_iter_count(&io->iter))
break;
io->bytes_done += ret;
- iov_iter_save_state(&s->iter, &s->iter_state);
+ iov_iter_save_state(&io->iter, &io->iter_state);
/* if we can retry, do so with the callbacks armed */
if (!io_rw_should_retry(req)) {
return -EAGAIN;
}
- req->cqe.res = iov_iter_count(&s->iter);
+ req->cqe.res = iov_iter_count(&io->iter);
/*
* Now retry read with the IOCB_WAITQ parts set in the iocb. If
* we get -EIOCBQUEUED, then we'll get a notification when the
* desired page gets unlocked. We can also get a partial read
* here, and if we do, then just retry at the new offset.
*/
- ret = io_iter_do_read(rw, &s->iter);
+ ret = io_iter_do_read(rw, &io->iter);
if (ret == -EIOCBQUEUED)
return IOU_ISSUE_SKIP_COMPLETE;
/* we got some bytes, but not all. retry. */
kiocb->ki_flags &= ~IOCB_WAITQ;
- iov_iter_restore(&s->iter, &s->iter_state);
+ iov_iter_restore(&io->iter, &io->iter_state);
} while (ret > 0);
done:
/* it's faster to check here then delegate to kfree */
- if (iovec)
- kfree(iovec);
return ret;
}
cflags = io_put_kbuf(req, issue_flags);
rw->len = 0; /* similarly to above, reset len to 0 */
- if (io_fill_cqe_req_aux(req,
- issue_flags & IO_URING_F_COMPLETE_DEFER,
- ret, cflags | IORING_CQE_F_MORE)) {
+ if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
if (issue_flags & IO_URING_F_MULTISHOT) {
/*
* Force retry, as we might have more data to
* multishot request, hitting overflow will terminate it.
*/
io_req_set_res(req, ret, cflags);
+ io_req_rw_cleanup(req, issue_flags);
if (issue_flags & IO_URING_F_MULTISHOT)
return IOU_STOP_MULTISHOT;
return IOU_OK;
int io_write(struct io_kiocb *req, unsigned int issue_flags)
{
+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
- struct io_rw_state __s, *s = &__s;
- struct iovec *iovec;
+ struct io_async_rw *io = req->async_data;
struct kiocb *kiocb = &rw->kiocb;
- bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
ssize_t ret, ret2;
loff_t *ppos;
- if (!req_has_async_data(req)) {
- ret = io_import_iovec(ITER_SOURCE, req, &iovec, s, issue_flags);
- if (unlikely(ret < 0))
- return ret;
- } else {
- struct io_async_rw *io = req->async_data;
-
- s = &io->s;
- iov_iter_restore(&s->iter, &s->iter_state);
- iovec = NULL;
- }
ret = io_rw_init_file(req, FMODE_WRITE);
- if (unlikely(ret)) {
- kfree(iovec);
+ if (unlikely(ret))
return ret;
- }
- req->cqe.res = iov_iter_count(&s->iter);
+ req->cqe.res = iov_iter_count(&io->iter);
if (force_nonblock) {
/* If the file doesn't support async, just async punt */
if (unlikely(!io_file_supports_nowait(req)))
- goto copy_iov;
+ goto ret_eagain;
- /* File path supports NOWAIT for non-direct_IO only for block devices. */
+ /* Check if we can support NOWAIT. */
if (!(kiocb->ki_flags & IOCB_DIRECT) &&
- !(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) &&
- (req->flags & REQ_F_ISREG))
+ !(req->file->f_op->fop_flags & FOP_BUFFER_WASYNC) &&
+ (req->flags & REQ_F_ISREG))
- goto copy_iov;
+ goto ret_eagain;
kiocb->ki_flags |= IOCB_NOWAIT;
} else {
ppos = io_kiocb_update_pos(req);
ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res);
- if (unlikely(ret)) {
- kfree(iovec);
+ if (unlikely(ret))
return ret;
- }
if (req->flags & REQ_F_ISREG)
kiocb_start_write(kiocb);
kiocb->ki_flags |= IOCB_WRITE;
if (likely(req->file->f_op->write_iter))
- ret2 = call_write_iter(req->file, kiocb, &s->iter);
+ ret2 = call_write_iter(req->file, kiocb, &io->iter);
else if (req->file->f_op->write)
- ret2 = loop_rw_iter(WRITE, rw, &s->iter);
+ ret2 = loop_rw_iter(WRITE, rw, &io->iter);
else
ret2 = -EINVAL;
if (!force_nonblock || ret2 != -EAGAIN) {
/* IOPOLL retry should happen for io-wq threads */
if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL))
- goto copy_iov;
+ goto ret_eagain;
if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) {
- struct io_async_rw *io;
-
trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,
req->cqe.res, ret2);
* in the worker. Also update bytes_done to account for
* the bytes already written.
*/
- iov_iter_save_state(&s->iter, &s->iter_state);
- ret = io_setup_async_rw(req, iovec, s, true);
-
- io = req->async_data;
- if (io)
- io->bytes_done += ret2;
+ iov_iter_save_state(&io->iter, &io->iter_state);
+ io->bytes_done += ret2;
if (kiocb->ki_flags & IOCB_WRITE)
io_req_end_write(req);
- return ret ? ret : -EAGAIN;
+ return -EAGAIN;
}
done:
- ret = kiocb_done(req, ret2, issue_flags);
+ return kiocb_done(req, ret2, issue_flags);
} else {
- copy_iov:
- iov_iter_restore(&s->iter, &s->iter_state);
- ret = io_setup_async_rw(req, iovec, s, false);
- if (!ret) {
- if (kiocb->ki_flags & IOCB_WRITE)
- io_req_end_write(req);
- return -EAGAIN;
- }
- return ret;
+ ret_eagain:
+ iov_iter_restore(&io->iter, &io->iter_state);
+ if (kiocb->ki_flags & IOCB_WRITE)
+ io_req_end_write(req);
+ return -EAGAIN;
}
- /* it's reportedly faster than delegating the null check to kfree() */
- if (iovec)
- kfree(iovec);
- return ret;
}
void io_rw_fail(struct io_kiocb *req)
break;
nr_events++;
req->cqe.flags = io_put_kbuf(req, 0);
+ if (req->opcode != IORING_OP_URING_CMD)
+ io_req_rw_cleanup(req, 0);
}
if (unlikely(!nr_events))
return 0;
__io_submit_flush_completions(ctx);
return nr_events;
}
+
+ void io_rw_cache_free(const void *entry)
+ {
+ struct io_async_rw *rw = (struct io_async_rw *) entry;
+
+ if (rw->free_iovec) {
+ kasan_mempool_unpoison_object(rw->free_iovec,
+ rw->free_iov_nr * sizeof(struct iovec));
+ io_rw_iovec_free(rw);
+ }
+ kfree(rw);
+ }
projects / linux.git / commitdiff