From 8db455c77265063a10a9d08147b8fd4ed20c59c4 Mon Sep 17 00:00:00 2001 From: Qian Yingjin Date: Mon, 27 Dec 2021 11:23:45 +0800 Subject: [PATCH] LU-15399 llite: dont restart directIO with IOCB_NOWAIT It should hanlde FLR mirror retry and io_uring with IOCB_NOWAIT flag differently. int cl_io_loop(const struct lu_env *env, struct cl_io *io) { ... if (result == -EAGAIN && io->ci_ndelay) { io->ci_need_restart = 1; result = 0; } ... } ssize_t generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) { ... if (iocb->ki_flags & IOCB_NOWAIT) { if (filemap_range_has_page(mapping, iocb->ki_pos, iocb->ki_pos + count - 1)) return -EAGAIN; ... } In current code, it will restart I/O engine for read when get -EAGAIN code. However, for io_uring direct IO with IOCB_NOWAIT, if found that there are cache pages in the current I/O range, it should return -EAGAIN to the upper layer immediately. Otherwise, it will stuck in an endless loop. This patch also adds a tool "io_uring_probe" to check whether the kernel supports io_uring fully. The reason adding this check is because the rhel8.5 kernel has backported io_uring: cat /proc/kallsyms |grep io_uring ffffffffa8510e10 W __x64_sys_io_uring_enter ffffffffa8510e10 W __x64_sys_io_uring_register ffffffffa8510e10 W __x64_sys_io_uring_setup but the io_uring syscalls return -ENOSYS. Test-Parameters: clientdistro=ubuntu2004 testlist=sanity-pcc Signed-off-by: Qian Yingjin Change-Id: Id4374382e56e90d02349676891aa57b216b3deff Reviewed-on: https://review.whamcloud.com/46147 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Patrick Farrell Reviewed-by: Oleg Drokin --- lustre/include/cl_object.h | 6 ++++- lustre/llite/file.c | 6 +++++ lustre/obdclass/cl_io.c | 2 +- lustre/tests/Makefile.am | 2 +- lustre/tests/io_uring_probe.c | 58 +++++++++++++++++++++++++++++++++++++++++++ lustre/tests/sanity-pcc.sh | 41 ++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 lustre/tests/io_uring_probe.c diff --git a/lustre/include/cl_object.h b/lustre/include/cl_object.h index 5f2c355..60f5f44 100644 --- a/lustre/include/cl_object.h +++ b/lustre/include/cl_object.h @@ -1952,7 +1952,11 @@ struct cl_io { /** * Bypass quota check */ - unsigned ci_noquota:1; + unsigned ci_noquota:1, + /** + * io_uring direct IO with flags IOCB_NOWAIT. + */ + ci_iocb_nowait:1; /** * How many times the read has retried before this one. * Set by the top level and consumed by the LOV. diff --git a/lustre/llite/file.c b/lustre/llite/file.c index bf2f49b..cbb9f15 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -1577,6 +1577,12 @@ void ll_io_init(struct cl_io *io, struct file *file, enum cl_io_type iot, #endif } +#ifdef IOCB_NOWAIT + io->ci_iocb_nowait = !!(args && + (args->u.normal.via_iocb->ki_flags & + IOCB_NOWAIT)); +#endif + io->ci_obj = ll_i2info(inode)->lli_clob; io->ci_lockreq = CILR_MAYBE; if (ll_file_nolock(file)) { diff --git a/lustre/obdclass/cl_io.c b/lustre/obdclass/cl_io.c index 6c7e7e4..18468e1 100644 --- a/lustre/obdclass/cl_io.c +++ b/lustre/obdclass/cl_io.c @@ -796,7 +796,7 @@ int cl_io_loop(const struct lu_env *env, struct cl_io *io) if (rc && !result) result = rc; - if (result == -EAGAIN && io->ci_ndelay) { + if (result == -EAGAIN && io->ci_ndelay && !io->ci_iocb_nowait) { io->ci_need_restart = 1; result = 0; } diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 069080b..202c99d 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -83,7 +83,7 @@ THETESTS += swap_lock_test lockahead_test mirror_io mmap_mknod_test THETESTS += create_foreign_file parse_foreign_file THETESTS += create_foreign_dir parse_foreign_dir THETESTS += check_fallocate splice-test lseek_test expand_truncate_test -THETESTS += foreign_symlink_striping lov_getstripe_old +THETESTS += foreign_symlink_striping lov_getstripe_old io_uring_probe if LIBAIO THETESTS += aiocp diff --git a/lustre/tests/io_uring_probe.c b/lustre/tests/io_uring_probe.c new file mode 100644 index 0000000..f96715f --- /dev/null +++ b/lustre/tests/io_uring_probe.c @@ -0,0 +1,58 @@ +/* + * LGPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the GNU Lesser General Public License + * (LGPL) version 2.1 or (at your discretion) any later version. + * (LGPL) version 2.1 accompanies this distribution, and is available at + * http://www.gnu.org/licenses/lgpl-2.1.html + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * LGPL HEADER END + */ +/* + * Copyright (c) 2022, DDN/Whamcloud Storage Corporation. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + */ +/* + * Probe whether OS supports io_uring. + * + * Author: Qian Yingjin + */ + +#include +#include +#include +#include +#include +#include + +#ifdef __NR_io_uring_register +#include + +int main(int argc, char **argv) +{ + int rc; + + rc = syscall(__NR_io_uring_register, 0, IORING_UNREGISTER_BUFFERS, + NULL, 0); + if (rc < 0 && errno == ENOSYS) { + printf("Your kernel does not support io_uring"); + return -ENOSYS; + } + return 0; +} +#else +int main(int argc, char **argv) +{ + return -ENOSYS; +} +#endif diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index 9ce8b47..30ca063 100644 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -1610,6 +1610,47 @@ test_101a() { } run_test 101a "Test auto attach in mount namespace (simulated container)" +test_102() { + grep -q io_uring_setup /proc/kallsyms || + skip "Client OS does not support io_uring I/O engine" + io_uring_probe || skip "kernel does not support io_uring fully" + + $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro || + skip "Server does not support PCC-RO" + + which fio || skip_env "no fio installed" + fio --enghelp | grep -q io_uring || + skip_env "fio does not support io_uring I/O engine" + + local loopfile="$TMP/$tfile" + local mntpt="/mnt/pcc.$tdir" + local hsm_root="$mntpt/$tdir" + local file=$DIR/$tfile + + setup_loopdev client $loopfile $mntpt 60 + mkdir $hsm_root || error "mkdir $hsm_root failed" + setup_pcc_mapping client \ + "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1" + do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0 + + local ioengine="io_uring" + local numjobs=2 + local size=10M + + do_facet $SINGLEAGT fio --name=seqwrite --ioengine=$ioengine \ + --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \ + --iodepth=64 --size=$size --filename=$file --rw=write || + error "fio seqwrite $file failed" + + # Read the file will trigger the buffered read from Lustre OSTs and + # write to PCC copy as @pcc_async_threshold is set with 0. + do_facet $SINGLEAGT fio --name=seqread --ioengine=$ioengine \ + --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \ + --iodepth=64 --size=$size --filename=$file --rw=read || + error "fio seqread $file failed" +} +run_test 102 "PCC-RO should not hange for io_uring I/O engine" + complete $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1