diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 3aa16e27f509..4289f4a92693 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -146,15 +147,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->flags |= REQ_F_BL_EMPTY; head &= bl->mask; - /* mmaped buffers are always contig */ - if (bl->is_mmap || head < IO_BUFFER_LIST_BUF_PER_PAGE) { - buf = &br->bufs[head]; - } else { - int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1); - int index = head / IO_BUFFER_LIST_BUF_PER_PAGE; - buf = page_address(bl->buf_pages[index]); - buf += off; - } + buf = &br->bufs[head]; if (*len == 0 || *len > buf->len) *len = buf->len; req->flags |= REQ_F_BUFFER_RING; @@ -241,6 +234,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, for (j = 0; j < bl->buf_nr_pages; j++) unpin_user_page(bl->buf_pages[j]); kvfree(bl->buf_pages); + vunmap(bl->buf_ring); bl->buf_pages = NULL; bl->buf_nr_pages = 0; } @@ -498,9 +492,9 @@ err: static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg, struct io_buffer_list *bl) { - struct io_uring_buf_ring *br; + struct io_uring_buf_ring *br = NULL; + int nr_pages, ret, i; struct page **pages; - int i, nr_pages; pages = io_pin_pages(reg->ring_addr, flex_array_size(br, bufs, reg->ring_entries), @@ -508,18 +502,12 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg, if (IS_ERR(pages)) return PTR_ERR(pages); - /* - * Apparently some 32-bit boxes (ARM) will return highmem pages, - * which then need to be mapped. We could support that, but it'd - * complicate the code and slowdown the common cases quite a bit. - * So just error out, returning -EINVAL just like we did on kernels - * that didn't support mapped buffer rings. - */ - for (i = 0; i < nr_pages; i++) - if (PageHighMem(pages[i])) - goto error_unpin; + br = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!br) { + ret = -ENOMEM; + goto error_unpin; + } - br = page_address(pages[0]); #ifdef SHM_COLOUR /* * On platforms that have specific aliasing requirements, SHM_COLOUR @@ -530,8 +518,10 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg, * should use IOU_PBUF_RING_MMAP instead, and liburing will handle * this transparently. */ - if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1)) + if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1)) { + ret = -EINVAL; goto error_unpin; + } #endif bl->buf_pages = pages; bl->buf_nr_pages = nr_pages; @@ -543,7 +533,8 @@ error_unpin: for (i = 0; i < nr_pages; i++) unpin_user_page(pages[i]); kvfree(pages); - return -EINVAL; + vunmap(br); + return ret; } /*