#define MAX_DIRECTIO_SIZE 2*1024*1024*1024UL
static inline int ll_get_user_pages(int rw, unsigned long user_addr,
- size_t size, struct page ***pages)
+ size_t size, struct page ***pages,
+ int *max_pages)
{
int result = -ENOMEM;
- int page_count;
/* set an arbitrary limit to prevent arithmetic overflow */
if (size > MAX_DIRECTIO_SIZE) {
return -EFBIG;
}
- page_count = (user_addr + size + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
- page_count -= user_addr >> CFS_PAGE_SHIFT;
+ *max_pages = (user_addr + size + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
+ *max_pages -= user_addr >> CFS_PAGE_SHIFT;
- OBD_ALLOC_WAIT(*pages, page_count * sizeof(**pages));
+ OBD_ALLOC_WAIT(*pages, *max_pages * sizeof(**pages));
if (*pages) {
down_read(¤t->mm->mmap_sem);
result = get_user_pages(current, current->mm, user_addr,
- page_count, (rw == READ), 0, *pages,
+ *max_pages, (rw == READ), 0, *pages,
NULL);
up_read(¤t->mm->mmap_sem);
- if (result < 0)
- OBD_FREE(*pages, page_count * sizeof(**pages));
+ if (unlikely(result <= 0))
+ OBD_FREE(*pages, *max_pages * sizeof(**pages));
}
return result;
int i;
for (i = 0; i < npages; i++) {
+ if (pages[i] == NULL)
+ break;
if (do_dirty)
set_page_dirty_lock(pages[i]);
page_cache_release(pages[i]);
struct ll_dio_pages *pv)
{
struct cl_page *clp;
- struct ccc_page *clup;
struct cl_2queue *queue;
struct cl_object *obj = io->ci_obj;
- struct cl_sync_io *anchor = &ccc_env_info(env)->cti_sync_io;
int i;
ssize_t rc = 0;
loff_t file_offset = pv->ldp_start_offset;
size_t page_size = cl_page_size(obj);
ENTRY;
- cl_sync_io_init(anchor, page_count);
-
queue = &io->ci_queue;
cl_2queue_init(queue);
for (i = 0; i < page_count; i++) {
break;
}
- clup = cl2ccc_page(cl_page_at(clp, &vvp_device_type));
- clup->cpg_sync_io = anchor;
cl_2queue_add(queue, clp);
/* drop the reference count for cl_page_find, so that the page
}
if (rc == 0) {
- rc = cl_io_submit_rw(env, io, rw == READ ? CRT_READ : CRT_WRITE,
- queue, CRP_NORMAL);
- if (rc == 0) {
- /*
- * If some pages weren't sent for any reason (e.g.,
- * direct-io read found up-to-date pages in the
- * cache), count them as completed to avoid infinite
- * wait.
- */
- cl_page_list_for_each(clp, &queue->c2_qin)
- cl_sync_io_note(anchor, +1);
- /* wait for the IO to be finished. */
- rc = cl_sync_io_wait(env, io, &queue->c2_qout,
- anchor) ?: pv->ldp_size;
- }
+ rc = cl_io_submit_sync(env, io,
+ rw == READ ? CRT_READ : CRT_WRITE,
+ queue, CRP_NORMAL, 0);
+ if (rc == 0)
+ rc = pv->ldp_size;
}
cl_2queue_discard(env, io, queue);
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_mapping->host;
struct ccc_object *obj = cl_inode2ccc(inode);
- ssize_t count = iov_length(iov, nr_segs), tot_bytes = 0;
+ ssize_t count = iov_length(iov, nr_segs);
+ ssize_t tot_bytes = 0, result = 0;
struct ll_inode_info *lli = ll_i2info(inode);
struct lov_stripe_md *lsm = lli->lli_smd;
unsigned long seg = 0;
while (iov_left > 0) {
struct page **pages;
- int page_count;
- ssize_t result;
+ int page_count, max_pages = 0;
+ size_t bytes;
+ bytes = min(size,iov_left);
page_count = ll_get_user_pages(rw, user_addr,
- min(size, iov_left),
- &pages);
- LASSERT(page_count != 0);
- if (page_count > 0) {
+ bytes,
+ &pages, &max_pages);
+ if (likely(page_count > 0)) {
+ if (unlikely(page_count < max_pages))
+ bytes = page_count << CFS_PAGE_SHIFT;
result = ll_direct_IO_26_seg(env, io, rw, inode,
file->f_mapping,
- min(size,iov_left),
+ bytes,
file_offset, pages,
page_count);
- ll_free_user_pages(pages, page_count, rw==READ);
+ ll_free_user_pages(pages, max_pages, rw==READ);
+ } else if (page_count == 0) {
+ GOTO(out, result = -EFAULT);
} else {
- result = 0;
+ result = page_count;
}
- if (page_count < 0 || result <= 0) {
+ if (unlikely(result <= 0)) {
/* If we can't allocate a large enough buffer
* for the request, shrink it to a smaller
* PAGE_SIZE multiple and try again.
* We should always be able to kmalloc for a
* page worth of page pointers = 4MB on i386. */
- if ((page_count == -ENOMEM||result == -ENOMEM)&&
+ if (result == -ENOMEM &&
size > (CFS_PAGE_SIZE / sizeof(*pages)) *
CFS_PAGE_SIZE) {
size = ((((size / 2) - 1) |
continue;
}
- if (tot_bytes <= 0)
- tot_bytes = page_count < 0 ? page_count : result;
- GOTO(out, tot_bytes);
+ GOTO(out, result);
}
tot_bytes += result;
}
cl_env_put(env, &refcheck);
- RETURN(tot_bytes);
+ RETURN(tot_bytes ? : result);
}
struct address_space_operations ll_aops = {